gh-117139: Convert the evaluation stack to stack refs (#118450)

This PR sets up tagged pointers for CPython.
The general idea is to create a separate struct _PyStackRef for everything on the evaluation stack to store the bits. This forces the C compiler to warn us if we try to cast things or pull things out of the struct directly.
Only for free threading: We tag the low bit if something is deferred - that means we skip incref and decref operations on it. This behavior may change in the future if Mark's plans to defer all objects in the interpreter loop pans out.
This implies a strict stack reference discipline is required. ALL incref and decref operations on stackrefs must use the stackref variants. It is unsafe to untag something then do normal incref/decref ops on it.
The new incref and decref variants are called dup and close. They mimic a "handle" API operating on these stackrefs.
Please read Include/internal/pycore_stackref.h for more information!
---------
Co-authored-by: Mark Shannon <9448417+markshannon@users.noreply.github.com>

Author: Fidget-Spinner

Include/internal/pycore_ceval.h

@@ -261,8 +261,11 @@ PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *c
 PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState*tstate, PyObject*func, PyObject*kwargs);
 PyAPI_FUNC(PyObject*)_PyEval_MatchClass(PyThreadState*tstate, PyObject*subject, PyObject*type, Py_ssize_tnargs, PyObject*kwargs);
 PyAPI_FUNC(PyObject*)_PyEval_MatchKeys(PyThreadState*tstate, PyObject*map, PyObject*keys);
-PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState*tstate, PyObject*v, intargcnt, intargcntafter, PyObject**sp);
+PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState*tstate, _PyStackRefv, intargcnt, intargcntafter, _PyStackRef*sp);
 PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState*tstate, _PyInterpreterFrame*frame);
+PyAPI_FUNC(PyObject**) _PyObjectArray_FromStackRefArray(_PyStackRef*input, Py_ssize_tnargs, PyObject**scratch);
+
+PyAPI_FUNC(void) _PyObjectArray_Free(PyObject**array, PyObject**scratch);
 
 
 /* Bits that can be set in PyThreadState.eval_breaker */

Include/internal/pycore_code.h

@@ -8,6 +8,7 @@ extern "C" {
 # error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include"pycore_stackref.h"// _PyStackRef
 #include"pycore_lock.h"// PyMutex
 #include"pycore_backoff.h"// _Py_BackoffCounter
 
@@ -317,30 +318,30 @@ extern void _PyCode_Clear_Executors(PyCodeObject *code);
 
 /* Specialization functions */
 
-externvoid_Py_Specialize_LoadSuperAttr(PyObject*global_super, PyObject*cls,
+externvoid_Py_Specialize_LoadSuperAttr(_PyStackRefglobal_super, _PyStackRefcls,
 _Py_CODEUNIT*instr, intload_method);
-externvoid_Py_Specialize_LoadAttr(PyObject*owner, _Py_CODEUNIT*instr,
+externvoid_Py_Specialize_LoadAttr(_PyStackRefowner, _Py_CODEUNIT*instr,
 PyObject*name);
-externvoid_Py_Specialize_StoreAttr(PyObject*owner, _Py_CODEUNIT*instr,
+externvoid_Py_Specialize_StoreAttr(_PyStackRefowner, _Py_CODEUNIT*instr,
 PyObject*name);
 externvoid_Py_Specialize_LoadGlobal(PyObject*globals, PyObject*builtins,
 _Py_CODEUNIT*instr, PyObject*name);
-externvoid_Py_Specialize_BinarySubscr(PyObject*sub, PyObject*container,
+externvoid_Py_Specialize_BinarySubscr(_PyStackRefsub, _PyStackRefcontainer,
 _Py_CODEUNIT*instr);
-externvoid_Py_Specialize_StoreSubscr(PyObject*container, PyObject*sub,
+externvoid_Py_Specialize_StoreSubscr(_PyStackRefcontainer, _PyStackRefsub,
 _Py_CODEUNIT*instr);
-externvoid_Py_Specialize_Call(PyObject*callable, _Py_CODEUNIT*instr,
+externvoid_Py_Specialize_Call(_PyStackRefcallable, _Py_CODEUNIT*instr,
 intnargs);
-externvoid_Py_Specialize_BinaryOp(PyObject*lhs, PyObject*rhs, _Py_CODEUNIT*instr,
-intoparg, PyObject**locals);
-externvoid_Py_Specialize_CompareOp(PyObject*lhs, PyObject*rhs,
+externvoid_Py_Specialize_BinaryOp(_PyStackReflhs, _PyStackRefrhs, _Py_CODEUNIT*instr,
+intoparg, _PyStackRef*locals);
+externvoid_Py_Specialize_CompareOp(_PyStackReflhs, _PyStackRefrhs,
 _Py_CODEUNIT*instr, intoparg);
-externvoid_Py_Specialize_UnpackSequence(PyObject*seq, _Py_CODEUNIT*instr,
+externvoid_Py_Specialize_UnpackSequence(_PyStackRefseq, _Py_CODEUNIT*instr,
 intoparg);
-externvoid_Py_Specialize_ForIter(PyObject*iter, _Py_CODEUNIT*instr, intoparg);
-externvoid_Py_Specialize_Send(PyObject*receiver, _Py_CODEUNIT*instr);
-externvoid_Py_Specialize_ToBool(PyObject*value, _Py_CODEUNIT*instr);
-externvoid_Py_Specialize_ContainsOp(PyObject*value, _Py_CODEUNIT*instr);
+externvoid_Py_Specialize_ForIter(_PyStackRefiter, _Py_CODEUNIT*instr, intoparg);
+externvoid_Py_Specialize_Send(_PyStackRefreceiver, _Py_CODEUNIT*instr);
+externvoid_Py_Specialize_ToBool(_PyStackRefvalue, _Py_CODEUNIT*instr);
+externvoid_Py_Specialize_ContainsOp(_PyStackRefvalue, _Py_CODEUNIT*instr);
 
 #ifdefPy_STATS
 

Include/internal/pycore_frame.h

@@ -11,6 +11,7 @@ extern "C" {
 #include<stdbool.h>
 #include<stddef.h>// offsetof()
 #include"pycore_code.h"// STATS
+#include"pycore_stackref.h"// _PyStackRef
 
 /* See Objects/frame_layout.md for an explanation of the frame stack
  * including explanation of the PyFrameObject and _PyInterpreterFrame
@@ -67,7 +68,7 @@ typedef struct _PyInterpreterFrame {
 uint16_treturn_offset; /* Only relevant during a function call */
 charowner;
 /* Locals and stack */
-PyObject*localsplus[1];
+_PyStackReflocalsplus[1];
 } _PyInterpreterFrame;
 
 #define_PyInterpreterFrame_LASTI(IF) \
@@ -78,23 +79,23 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
 return (PyCodeObject*)f->f_executable;
 }
 
-staticinlinePyObject**_PyFrame_Stackbase(_PyInterpreterFrame*f) {
-returnf->localsplus+_PyFrame_GetCode(f)->co_nlocalsplus;
+staticinline_PyStackRef*_PyFrame_Stackbase(_PyInterpreterFrame*f) {
+return(f->localsplus+_PyFrame_GetCode(f)->co_nlocalsplus);
 }
 
-staticinlinePyObject*_PyFrame_StackPeek(_PyInterpreterFrame*f) {
+staticinline_PyStackRef_PyFrame_StackPeek(_PyInterpreterFrame*f) {
 assert(f->stacktop>_PyFrame_GetCode(f)->co_nlocalsplus);
-assert(f->localsplus[f->stacktop-1]!=NULL);
+assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1]));
 returnf->localsplus[f->stacktop-1];
 }
 
-staticinlinePyObject*_PyFrame_StackPop(_PyInterpreterFrame*f) {
+staticinline_PyStackRef_PyFrame_StackPop(_PyInterpreterFrame*f) {
 assert(f->stacktop>_PyFrame_GetCode(f)->co_nlocalsplus);
 f->stacktop--;
 returnf->localsplus[f->stacktop];
 }
 
-staticinlinevoid_PyFrame_StackPush(_PyInterpreterFrame*f, PyObject*value) {
+staticinlinevoid_PyFrame_StackPush(_PyInterpreterFrame*f, _PyStackRefvalue) {
 f->localsplus[f->stacktop] =value;
 f->stacktop++;
 }
@@ -143,14 +144,14 @@ _PyFrame_Initialize(
 frame->owner=FRAME_OWNED_BY_THREAD;
 
 for (inti=null_locals_from; i<code->co_nlocalsplus; i++) {
-frame->localsplus[i] =NULL;
+frame->localsplus[i] =PyStackRef_NULL;
  }
 }
 
 /* Gets the pointer to the locals array
  * that precedes this frame.
  */
-staticinlinePyObject**
+staticinline_PyStackRef*
 _PyFrame_GetLocalsArray(_PyInterpreterFrame*frame)
 {
 returnframe->localsplus;
@@ -160,16 +161,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
  Having stacktop <= 0 ensures that invalid
  values are not visible to the cycle GC.
  We choose -1 rather than 0 to assist debugging. */
-staticinlinePyObject**
+staticinline_PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame*frame)
 {
-PyObject**sp=frame->localsplus+frame->stacktop;
+_PyStackRef*sp=frame->localsplus+frame->stacktop;
 frame->stacktop=-1;
 returnsp;
 }
 
 staticinlinevoid
-_PyFrame_SetStackPointer(_PyInterpreterFrame*frame, PyObject**stack_pointer)
+_PyFrame_SetStackPointer(_PyInterpreterFrame*frame, _PyStackRef*stack_pointer)
 {
 frame->stacktop= (int)(stack_pointer-frame->localsplus);
 }
@@ -309,7 +310,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
 
 PyAPI_FUNC(_PyInterpreterFrame*)
 _PyEvalFramePushAndInit(PyThreadState*tstate, PyFunctionObject*func,
-PyObject*locals, PyObject*const*args,
+PyObject*locals, _PyStackRefconst*args,
 size_targcount, PyObject*kwnames);
 
 #ifdef__cplusplus

Include/internal/pycore_jit.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #ifdef_Py_JIT
 
-typedef_Py_CODEUNIT*(*jit_func)(_PyInterpreterFrame*frame, PyObject**stack_pointer, PyThreadState*tstate);
+typedef_Py_CODEUNIT*(*jit_func)(_PyInterpreterFrame*frame, _PyStackRef*stack_pointer, PyThreadState*tstate);
 
 int_PyJIT_Compile(_PyExecutorObject*executor, const_PyUOpInstruction*trace, size_tlength);
 void_PyJIT_Free(_PyExecutorObject*executor);

Include/internal/pycore_object.h

@@ -159,21 +159,6 @@ static inline void _Py_ClearImmortal(PyObject *op)
  op = NULL; \
  } while (0)
 
-// Mark an object as supporting deferred reference counting. This is a no-op
-// in the default (with GIL) build. Objects that use deferred reference
-// counting should be tracked by the GC so that they are eventually collected.
-externvoid_PyObject_SetDeferredRefcount(PyObject*op);
-
-staticinlineint
-_PyObject_HasDeferredRefcount(PyObject*op)
-{
-#ifdefPy_GIL_DISABLED
-return_PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED);
-#else
-return0;
-#endif
-}
-
 #if !defined(Py_GIL_DISABLED)
 staticinlinevoid
 _Py_DECREF_SPECIALIZED(PyObject*op, constdestructordestruct)

Include/internal/pycore_object_deferred.h

@@ -0,0 +1,32 @@
+#ifndefPy_INTERNAL_OBJECT_DEFERRED_H
+#definePy_INTERNAL_OBJECT_DEFERRED_H
+
+#ifdef__cplusplus
+extern"C" {
+#endif
+
+#include"pycore_gc.h"
+
+#ifndefPy_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Mark an object as supporting deferred reference counting. This is a no-op
+// in the default (with GIL) build. Objects that use deferred reference
+// counting should be tracked by the GC so that they are eventually collected.
+externvoid_PyObject_SetDeferredRefcount(PyObject*op);
+
+staticinlineint
+_PyObject_HasDeferredRefcount(PyObject*op)
+{
+#ifdefPy_GIL_DISABLED
+return_PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED);
+#else
+return0;
+#endif
+}
+
+#ifdef__cplusplus
+}
+#endif
+#endif// !Py_INTERNAL_OBJECT_DEFERRED_H

Include/internal/pycore_optimizer.h

@@ -271,7 +271,7 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx);
 
 PyAPI_FUNC(PyObject*) _Py_uop_symbols_test(PyObject*self, PyObject*ignored);
 
-PyAPI_FUNC(int) _PyOptimizer_Optimize(struct_PyInterpreterFrame*frame, _Py_CODEUNIT*start, PyObject**stack_pointer, _PyExecutorObject**exec_ptr);
+PyAPI_FUNC(int) _PyOptimizer_Optimize(struct_PyInterpreterFrame*frame, _Py_CODEUNIT*start, _PyStackRef*stack_pointer, _PyExecutorObject**exec_ptr);
 
 #ifdef__cplusplus
 }