gh-103743: Add PyUnstable_Object_GC_NewWithExtraData (#103743)

Author: jbradaric

Doc/c-api/gcsupport.rst

@@ -65,6 +65,19 @@ rules:
  Analogous to :c:func:`PyObject_NewVar` but for container objects with the
 :const:`Py_TPFLAGS_HAVE_GC` flag set.
 
+.. c:function::TYPE* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size)
+
+ Analogous to :c:func:`PyObject_GC_New` but for container objects that
+ have additional data at the end of the object not managed by Python.
+
+ .. warning::
+ The function is marked as unstable because the final mechanism
+ for reserving extra data after an instance is not yet decided.
+ Once :pep:`697` is implemented, the mechanism described there can
+ be used to reserve the extra data.
+
+ .. versionadded:: 3.12
+
 
 .. c:function::TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize)
 

Include/objimpl.h

@@ -131,6 +131,10 @@ PyAPI_FUNC(PyVarObject *) PyObject_InitVar(PyVarObject *,
 PyAPI_FUNC(PyObject*) _PyObject_New(PyTypeObject*);
 PyAPI_FUNC(PyVarObject*) _PyObject_NewVar(PyTypeObject*, Py_ssize_t);
 
+#if !defined(Py_LIMITED_API)
+PyAPI_FUNC(PyObject*) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject*, size_t);
+#endif
+
 #definePyObject_New(type, typeobj) ((type *)_PyObject_New(typeobj))
 
 // Alias to PyObject_New(). In Python 3.8, PyObject_NEW() called directly

Lib/test/test_capi/test_misc.py

@@ -1043,6 +1043,20 @@ class dictsub(dict): ... # dict subclasses must work
 self.assertEqual(_testcapi.function_get_kw_defaults(some), None)
 self.assertEqual(some.__kwdefaults__, None)
 
+deftest_unstable_gc_new_with_extra_data(self):
+classData(_testcapi.ObjExtraData):
+__slots__= ('x', 'y')
+
+d=Data()
+d.x=10
+d.y=20
+d.extra=30
+self.assertEqual(d.x, 10)
+self.assertEqual(d.y, 20)
+self.assertEqual(d.extra, 30)
+deld.extra
+self.assertIsNone(d.extra)
+
 
 classTestPendingCalls(unittest.TestCase):
 

Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst

@@ -0,0 +1,2 @@
+Add ``PyUnstable_Object_GC_NewWithExtraData`` function that can be used to
+allocate additional memory after an object for data not managed by Python.

Modules/_testcapimodule.c

@@ -3343,7 +3343,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self),
  }
 state.target=obj;
 state.found=0;
-
+
 PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state);
 Py_DECREF(obj);
 if (!state.found) {
@@ -3380,6 +3380,94 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self),
 Py_RETURN_NONE;
 }
 
+typedefstruct {
+PyObject_HEAD
+} ObjExtraData;
+
+staticPyObject*
+obj_extra_data_new(PyTypeObject*type, PyObject*args, PyObject*kwds)
+{
+size_textra_size=sizeof(PyObject*);
+PyObject*obj=PyUnstable_Object_GC_NewWithExtraData(type, extra_size);
+if (obj==NULL)
+returnPyErr_NoMemory();
+memset(obj, '\0', type->tp_basicsize+extra_size);
+PyObject_Init(obj, type);
+PyObject_GC_Track(obj);
+returnobj;
+}
+
+staticPyObject**
+obj_extra_data_get_extra_storage(PyObject*self)
+{
+return (PyObject**)((char*)self+Py_TYPE(self)->tp_basicsize);
+}
+
+staticPyObject*
+obj_extra_data_get(PyObject*self, void*Py_UNUSED(ignored))
+{
+PyObject**extra_storage=obj_extra_data_get_extra_storage(self);
+PyObject*value=*extra_storage;
+if (!value)
+Py_RETURN_NONE;
+Py_INCREF(value);
+returnvalue;
+}
+
+staticint
+obj_extra_data_set(PyObject*self, PyObject*newval, void*Py_UNUSED(ignored))
+{
+PyObject**extra_storage=obj_extra_data_get_extra_storage(self);
+Py_CLEAR(*extra_storage);
+if (newval) {
+Py_INCREF(newval);
+*extra_storage=newval;
+ }
+return0;
+}
+
+staticPyGetSetDefobj_extra_data_getset[] = {
+ {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL},
+};
+
+staticint
+obj_extra_data_traverse(PyObject*self, visitprocvisit, void*arg)
+{
+PyObject**extra_storage=obj_extra_data_get_extra_storage(self);
+PyObject*value=*extra_storage;
+Py_VISIT(value);
+return0;
+}
+
+staticint
+obj_extra_data_clear(PyObject*self)
+{
+PyObject**extra_storage=obj_extra_data_get_extra_storage(self);
+Py_CLEAR(*extra_storage);
+return0;
+}
+
+staticvoid
+obj_extra_data_dealloc(PyObject*self)
+{
+PyObject_GC_UnTrack(self);
+obj_extra_data_clear(self);
+Py_TYPE(self)->tp_free(self);
+}
+
+staticPyTypeObjectObjExtraData_Type= {
+PyVarObject_HEAD_INIT(NULL, 0)
+"obj_with_extra_data",
+sizeof(ObjExtraData),
+0,
+ .tp_getset=obj_extra_data_getset,
+ .tp_dealloc=obj_extra_data_dealloc,
+ .tp_flags=Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ .tp_traverse= (traverseproc)obj_extra_data_traverse,
+ .tp_clear= (inquiry)obj_extra_data_clear,
+ .tp_new=obj_extra_data_new,
+ .tp_free=PyObject_GC_Del,
+};
 
 structatexit_data {
 intcalled;
@@ -4103,6 +4191,11 @@ PyInit__testcapi(void)
 Py_INCREF(&MethStatic_Type);
 PyModule_AddObject(m, "MethStatic", (PyObject*)&MethStatic_Type);
 
+if (PyType_Ready(&ObjExtraData_Type) <0)
+returnNULL;
+Py_INCREF(&ObjExtraData_Type);
+PyModule_AddObject(m, "ObjExtraData", (PyObject*)&ObjExtraData_Type);
+
 PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
 PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
 PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));

Modules/gcmodule.c

@@ -2357,6 +2357,18 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
 returnop;
 }
 
+PyObject*
+PyUnstable_Object_GC_NewWithExtraData(PyTypeObject*tp, size_textra_size)
+{
+size_tpresize=_PyType_PreHeaderSize(tp);
+PyObject*op=gc_alloc(_PyObject_SIZE(tp) +extra_size, presize);
+if (op==NULL) {
+returnNULL;
+ }
+_PyObject_Init(op, tp);
+returnop;
+}
+
 PyVarObject*
 _PyObject_GC_Resize(PyVarObject*op, Py_ssize_tnitems)
 {

Tools/c-analyzer/cpython/ignored.tsv

@@ -508,6 +508,7 @@ Modules/_testcapimodule.c	-	g_dict_watch_events	-
 Modules/_testcapimodule.c	-	g_dict_watchers_installed	-
 Modules/_testcapimodule.c	-	g_type_modified_events	-
 Modules/_testcapimodule.c	-	g_type_watchers_installed	-
+Modules/_testcapimodule.c	-	ObjExtraData_Type	-
 Modules/_testimportmultiple.c	-	_barmodule	-
 Modules/_testimportmultiple.c	-	_foomodule	-
 Modules/_testimportmultiple.c	-	_testimportmultiple	-