gh-103509: PEP 697 -- Limited C API for Extending Opaque Types

Doc/c-api/object.rst

@@ -395,3 +395,42 @@ Object Protocol
  returns ``NULL`` if the object cannot be iterated.
 
  .. versionadded:: 3.10
+
+.. c:function:: void *PyObject_GetTypeData(PyObject *o, PyTypeObject *cls)
+
+ Get a pointer to subclass-specific data reserved for *cls*.
+
+ The object *o* must be an instance of *cls*, and *cls* must have been
+ created using negative :c:member:`PyType_Spec.basicsize`.
+ Python does not check this.
+
+ On error, set an exception and return ``NULL``.
+
+ .. versionadded:: 3.12
+
+.. c:function:: Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls)
+
+ Return the size of the instance memory space reserved for *cls*, i.e. the size of the
+ memory :c:func:`PyObject_GetTypeData` returns.
+
+ This may be larger than requested using :c:member:`-PyType_Spec.basicsize <PyType_Spec.basicsize>`;
+ it is safe to use this larger size (e.g. with :c:func:`!memset`).
+
+ The type *cls* **must** have been created using
+ negative :c:member:`PyType_Spec.basicsize`.
+ Python does not check this.
+
+ On error, set an exception and return a negative value.
+
+ .. versionadded:: 3.12
+
+.. c:function:: void *PyObject_GetItemData(PyObject *o)
+
+ Get a pointer to per-item data for a class with
+ :const:`Py_TPFLAGS_ITEMS_AT_END`.
+
+ On error, set an exception and return ``NULL``.
+ :py:exc:`TypeError` is raised if *o* does not have
+ :const:`Py_TPFLAGS_ITEMS_AT_END` set.
+
+ .. versionadded:: 3.12

Doc/c-api/structures.rst

@@ -486,6 +486,22 @@ The following flags can be used with :c:member:`PyMemberDef.flags`:
  Emit an ``object.__getattr__`` :ref:`audit event <audit-events>`
  before reading.
 
+.. c:macro:: Py_RELATIVE_OFFSET
+
+ Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef``
+ entry indicates an offset from the subclass-specific data, rather than
+ from ``PyObject``.
+
+ Can only be used as part of :c:member:`Py_tp_members <PyTypeObject.tp_members>`
+ :c:type:`slot <PyTypeSlot>` when creating a class using negative
+ :c:member:`~PyTypeDef.basicsize`.
+ It is mandatory in that case.
+
+ This flag is only used in :c:type:`PyTypeSlot`.
+ When setting :c:member:`~PyTypeObject.tp_members` during
+ class creation, Python clears it and sets
+ :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct.
+
 .. index::
  single: READ_RESTRICTED
  single: WRITE_RESTRICTED

Doc/c-api/type.rst

@@ -353,25 +353,57 @@ The following functions and structs are used to create
 
  Structure defining a type's behavior.
 
- .. c:member:: const char* PyType_Spec.name
+ .. c:member:: const char* name
 
  Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
 
- .. c:member:: int PyType_Spec.basicsize
- .. c:member:: int PyType_Spec.itemsize
+ .. c:member:: int basicsize
 
- Size of the instance in bytes, used to set
- :c:member:`PyTypeObject.tp_basicsize` and
- :c:member:`PyTypeObject.tp_itemsize`.
+ If positive, specifies the size of the instance in bytes.
+ It is used to set :c:member:`PyTypeObject.tp_basicsize`.
 
- .. c:member:: int PyType_Spec.flags
+ If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize`
+ should be inherited.
+
+ If negative, the absolute value specifies how much space instances of the
+ class need *in addition* to the superclass.
+ Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific
+ memory reserved this way.
+
+ .. versionchanged:: 3.12
+
+ Previously, this field could not be negative.
+
+ .. c:member:: int itemsize
+
+ Size of one element of a variable-size type, in bytes.
+ Used to set :c:member:`PyTypeObject.tp_itemsize`.
+ See ``tp_itemsize`` documentation for caveats.
+
+ If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited.
+ Extending arbitrary variable-sized classes is dangerous,
+ since some types use a fixed offset for variable-sized memory,
+ which can then overlap fixed-sized memory used by a subclass.
+ To help prevent mistakes, inheriting ``itemsize`` is only possible
+ in the following situations:
+
+ - The base is not variable-sized (its
+ :c:member:`~PyTypeObject.tp_itemsize`).
+ - The requested :c:member:`PyType_Spec.basicsize` is positive,
+ suggesting that the memory layout of the base class is known.
+ - The requested :c:member:`PyType_Spec.basicsize` is zero,
+ suggesting that the subclass does not access the instance's memory
+ directly.
+ - With the :const:`Py_TPFLAGS_ITEMS_AT_END` flag.
+
+ .. c:member:: unsigned int flags
 
  Type flags, used to set :c:member:`PyTypeObject.tp_flags`.
 
  If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set,
  :c:func:`PyType_FromSpecWithBases` sets it automatically.
 
- .. c:member:: PyType_Slot *PyType_Spec.slots
+ .. c:member:: PyType_Slot *slots
 
  Array of :c:type:`PyType_Slot` structures.
  Terminated by the special slot value ``{0, NULL}``.

Doc/c-api/typeobj.rst

@@ -1171,6 +1171,26 @@ and :c:type:`PyType_Type` effectively act as defaults.)
  :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass.
 
 
+ .. data:: Py_TPFLAGS_ITEMS_AT_END
+
+ Only usable with variable-size types, i.e. ones with non-zero
+ :c:member:`~PyObject.tp_itemsize`.
+
+ Indicates that the variable-sized portion of an instance of this type is
+ at the end of the instance's memory area, at an offset of
+ :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each
+ subclass).
+
+ When setting this flag, be sure that all superclasses either
+ use this memory layout, or are not variable-sized.
+ Python does not check this.
+
+ .. versionadded:: 3.12
+
+ **Inheritance:**
+
+ This flag is inherited.
+
  .. XXX Document more flags here?
 
 

Doc/whatsnew/3.12.rst

@@ -1153,6 +1153,21 @@ New Features
 
  (Contributed by Petr Viktorin in :gh:`101101`.)
 
+* :pep:`697`: Added API for extending types whose instance memory layout is
+ opaque:
+
+ - :c:member:`PyType_Spec.basicsize` can be zero or negative to specify
+ inheriting or extending the base class size.
+ - :c:func:`PyObject_GetTypeData` and :c:func:`PyType_GetTypeDataSize`
+ added to allow access to subclass-specific instance data.
+ - :const:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData`
+ added to allow safely extending certain variable-sized types, including
+ :c:var:`PyType_Type`.
+ - :c:macro:`Py_RELATIVE_OFFSET` added to allow defining
+ :c:type:`members <PyMemberDef>` in terms of a subclass-specific struct.
+
+ (Contributed by Petr Viktorin in :gh:`103509`.)
+
 * Added the new limited C API function :c:func:`PyType_FromMetaclass`,
  which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using
  an additional metaclass argument.

Include/cpython/object.h

@@ -553,6 +553,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro;
  Py_TRASHCAN_END; \
  } while(0);
 
+PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
 
 PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg);
 PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj);

Include/descrobject.h

@@ -83,6 +83,7 @@ struct PyMemberDef {
 #define Py_READONLY 1
 #define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that
 #define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value.
+#define Py_RELATIVE_OFFSET 8
 
 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *);
 PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *);

Include/internal/pycore_object.h

@@ -389,11 +389,6 @@ extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
 extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
 extern int _PyObject_IsInstanceDictEmpty(PyObject *);
 
-// Access macro to the members which are floating "behind" the object
-static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) {
- return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize);
-}
-
 PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *);
 
 /* C function call trampolines to mitigate bad function pointer casts.

Include/object.h

@@ -355,6 +355,8 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *);
 #endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000
 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*);
+PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls);
+PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls);
 #endif
 
 /* Generic type check */
@@ -521,6 +523,9 @@ given type object has a specified feature.
 // subject itself (rather than a mapped attribute on it):
 #define _Py_TPFLAGS_MATCH_SELF (1UL << 22)
 
+/* Items (ob_size*tp_itemsize) are found at the end of an instance's memory */
+#define Py_TPFLAGS_ITEMS_AT_END (1UL << 23)
+
 /* These flags are used to determine if a type is a subclass. */
 #define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24)
 #define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25)

Include/pyport.h

@@ -765,4 +765,15 @@ extern char * _getpty(int *, int, mode_t, int);
 #undef __bool__
 #endif
 
+// Make sure we have maximum alignment, even if the current compiler
+// does not support max_align_t. Note that:
+// - Autoconf reports alignment of unknown types to 0.
+// - 'long double' has maximum alignment on *most* platforms,
+// looks like the best we can do for pre-C11 compilers.
+// - The value is tested, see test_alignof_max_align_t
+#if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0
+# undef ALIGNOF_MAX_ALIGN_T
+# define ALIGNOF_MAX_ALIGN_T _Alignof(long double)
+#endif
+
 #endif /* Py_PYPORT_H */