gh-148871: make LOAD_COMMON_CONSTANT use immortal stackref borrows (GH-149625)

Author: NekoAsakura

Include/internal/pycore_interp_structs.h

@@ -1001,7 +1001,7 @@ struct _is {
 structast_stateast;
 structtypes_statetypes;
 structcallable_cachecallable_cache;
-PyObject*common_consts[NUM_COMMON_CONSTANTS];
+_PyStackRefcommon_consts[NUM_COMMON_CONSTANTS];
 booljit;
 boolcompiling;
 

Include/internal/pycore_stackref.h

@@ -263,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 }
 #definePyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
 
+staticinline_PyStackRef
+_PyStackRef_DupImmortal(_PyStackRefref, constchar*filename, intlinenumber)
+{
+assert(!PyStackRef_IsError(ref));
+assert(!PyStackRef_IsTaggedInt(ref));
+assert(!PyStackRef_RefcountOnObject(ref));
+PyObject*obj=_Py_stackref_get_object(ref);
+assert(_Py_IsImmortal(obj));
+return_Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber);
+}
+#definePyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__)
+
 staticinlinevoid
 _PyStackRef_CLOSE_SPECIALIZED(_PyStackRefref, destructordestruct, constchar*filename, intlinenumber)
 {
@@ -633,6 +645,15 @@ PyStackRef_DUP(_PyStackRef ref)
 }
 #endif
 
+staticinline_PyStackRef
+PyStackRef_DupImmortal(_PyStackRefref)
+{
+assert(!PyStackRef_IsNull(ref));
+assert(!PyStackRef_RefcountOnObject(ref));
+assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref)));
+returnref;
+}
+
 staticinlinebool
 PyStackRef_IsHeapSafe(_PyStackRefref)
 {

Modules/_testinternalcapi/test_cases.c.h

@@ -1974,7 +1974,7 @@ dummy_func(
 inst(LOAD_COMMON_CONSTANT, ( --value)) {
 // Keep in sync with _common_constants in opcode.py
 assert(oparg<NUM_COMMON_CONSTANTS);
-value=PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
+value=PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]);
  }
 
 inst(LOAD_BUILD_CLASS, ( --bc)) {

Python/bytecodes.c

@@ -11,6 +11,7 @@
 #include"pycore_opcode_utils.h"
 #include"pycore_opcode_metadata.h"// OPCODE_HAS_ARG, etc
 #include"pycore_pystate.h"// _PyInterpreterState_GET()
+#include"pycore_stackref.h"// PyStackRef_AsPyObjectBorrow()
 
 #include<stdbool.h>
 
@@ -1330,7 +1331,8 @@ get_const_value(int opcode, int oparg, PyObject *co_consts)
  }
 if (opcode==LOAD_COMMON_CONSTANT) {
 assert(oparg<NUM_COMMON_CONSTANTS);
-returnPy_NewRef(_PyInterpreterState_GET()->common_consts[oparg]);
+returnPyStackRef_AsPyObjectBorrow(
+_PyInterpreterState_GET()->common_consts[oparg]);
  }
 
 if (constant==NULL) {

Python/executor_cases.c.h

@@ -2,6 +2,7 @@
 #include"pycore_long.h"
 #include"pycore_opcode_utils.h"
 #include"pycore_optimizer.h"
+#include"pycore_stackref.h"
 #include"pycore_typeobject.h"
 #include"pycore_uops.h"
 #include"pycore_uop_ids.h"
@@ -870,15 +871,11 @@ dummy_func(void) {
 
 op(_LOAD_COMMON_CONSTANT, (--value)) {
 assert(oparg<NUM_COMMON_CONSTANTS);
-PyObject*val=_PyInterpreterState_GET()->common_consts[oparg];
-if (_Py_IsImmortal(val)) {
-ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
-value=PyJitRef_Borrow(sym_new_const(ctx, val));
- }
-else {
-ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val);
-value=sym_new_const(ctx, val);
- }
+PyObject*val=PyStackRef_AsPyObjectBorrow(
+_PyInterpreterState_GET()->common_consts[oparg]);
+assert(_Py_IsImmortal(val));
+ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
+value=PyJitRef_Borrow(sym_new_const(ctx, val));
  }
 
 op(_LOAD_SMALL_INT, (--value)) {

Python/flowgraph.c

@@ -28,6 +28,7 @@
 #include"pycore_runtime.h"// _Py_ID()
 #include"pycore_runtime_init.h"// _PyRuntimeState_INIT
 #include"pycore_setobject.h"// _PySet_NextEntry()
+#include"pycore_stackref.h"// PyStackRef_FromPyObjectBorrow()
 #include"pycore_stats.h"// _PyStats_InterpInit()
 #include"pycore_sysmodule.h"// _PySys_ClearAttrString()
 #include"pycore_traceback.h"// PyUnstable_TracebackThreads()
@@ -878,25 +879,28 @@ pycore_init_builtins(PyThreadState *tstate)
  goto error;
  }
 
-interp->common_consts[CONSTANT_ASSERTIONERROR] =PyExc_AssertionError;
-interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] =PyExc_NotImplementedError;
-interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject*)&PyTuple_Type;
-interp->common_consts[CONSTANT_BUILTIN_ALL] =all;
-interp->common_consts[CONSTANT_BUILTIN_ANY] =any;
-interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject*)&PyList_Type;
-interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject*)&PySet_Type;
-interp->common_consts[CONSTANT_NONE] =Py_None;
-interp->common_consts[CONSTANT_EMPTY_STR] =
+PyObject*common_objs[NUM_COMMON_CONSTANTS] = {NULL};
+common_objs[CONSTANT_ASSERTIONERROR] =PyExc_AssertionError;
+common_objs[CONSTANT_NOTIMPLEMENTEDERROR] =PyExc_NotImplementedError;
+common_objs[CONSTANT_BUILTIN_TUPLE] = (PyObject*)&PyTuple_Type;
+common_objs[CONSTANT_BUILTIN_ALL] =all;
+common_objs[CONSTANT_BUILTIN_ANY] =any;
+common_objs[CONSTANT_BUILTIN_LIST] = (PyObject*)&PyList_Type;
+common_objs[CONSTANT_BUILTIN_SET] = (PyObject*)&PySet_Type;
+common_objs[CONSTANT_NONE] =Py_None;
+common_objs[CONSTANT_EMPTY_STR] =
 Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR);
-interp->common_consts[CONSTANT_TRUE] =Py_True;
-interp->common_consts[CONSTANT_FALSE] =Py_False;
-interp->common_consts[CONSTANT_MINUS_ONE] =
+common_objs[CONSTANT_TRUE] =Py_True;
+common_objs[CONSTANT_FALSE] =Py_False;
+common_objs[CONSTANT_MINUS_ONE] =
  (PyObject*)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS-1];
-interp->common_consts[CONSTANT_BUILTIN_FROZENSET] = (PyObject*)&PyFrozenSet_Type;
-interp->common_consts[CONSTANT_EMPTY_TUPLE] =
+common_objs[CONSTANT_BUILTIN_FROZENSET] = (PyObject*)&PyFrozenSet_Type;
+common_objs[CONSTANT_EMPTY_TUPLE] =
 Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE);
 for (inti=0; i<NUM_COMMON_CONSTANTS; i++) {
-assert(interp->common_consts[i] !=NULL);
+assert(common_objs[i] !=NULL);
+_Py_SetImmortal(common_objs[i]);
+interp->common_consts[i] =PyStackRef_FromPyObjectBorrow(common_objs[i]);
  }
 
 PyObject*list_append=_PyType_Lookup(&PyList_Type, &_Py_ID(append));