gh-103968: Deprecate creating heap types whose metaclass has custom tp_new.

Doc/c-api/type.rst

@@ -256,8 +256,13 @@ The following functions and structs are used to create
  The metaclass *metaclass* is used to construct the resulting type object.
  When *metaclass* is ``NULL``, the metaclass is derived from *bases*
  (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below).
- Note that metaclasses that override
- :c:member:`~PyTypeObject.tp_new` are not supported.
+
+ Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not
+ supported.
+ (For backwards compatibility, other ``PyType_From*`` functions allow
+ such metaclasses. They ignore ``tp_new``, which may result in incomplete
+ initialization. This is deprecated and in Python 3.14+ such metaclasses will
+ not be supported.)
 
  The *bases* argument can be used to specify base classes; it can either
  be only one class or a tuple of classes.
@@ -305,6 +310,11 @@ The following functions and structs are used to create
  The function now finds and uses a metaclass corresponding to the provided
  base classes. Previously, only :class:`type` instances were returned.
 
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
 
 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 
@@ -317,6 +327,12 @@ The following functions and structs are used to create
  The function now finds and uses a metaclass corresponding to the provided
  base classes. Previously, only :class:`type` instances were returned.
 
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
+
 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
 
  Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``.
@@ -327,6 +343,12 @@ The following functions and structs are used to create
  base classes provided in *Py_tp_base[s]* slots.
  Previously, only :class:`type` instances were returned.
 
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
+
 .. c:type:: PyType_Spec
 
  Structure defining a type's behavior.

Doc/whatsnew/3.12.rst

@@ -1308,6 +1308,21 @@ Porting to Python 3.12
  available on debug builds. If you happen to be using it then you'll
  need to start using ``_Py_GetGlobalRefTotal()``.
 
+* The following functions now select an appropriate metaclass for the newly
+ created type:
+
+ * :c:func:`PyType_FromSpec`
+ * :c:func:`PyType_FromSpecWithBases`
+ * :c:func:`PyType_FromModuleAndSpec`
+
+ Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new`
+ is deprecated, and in Python 3.14+ it will be disallowed.
+ Note that these functions ignore ``tp_new`` of the metaclass, possibly
+ allowing incomplete initialization.
+
+ Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12)
+ already disallows creating classes whose metaclass overrides ``tp_new``.
+
 Deprecated
 ----------
 
@@ -1384,6 +1399,11 @@ Deprecated
 * ``_PyErr_ChainExceptions`` is deprecated. Use ``_PyErr_ChainExceptions1``
  instead. (Contributed by Irit Katriel in :gh:`102192`.)
 
+* Using :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases`
+ or :c:func:`PyType_FromModuleAndSpec` to create a class whose metaclass
+ overrides :c:member:`~PyTypeObject.tp_new` is deprecated.
+ Call the metaclass instead.
+
 Removed
 -------
 

Lib/test/test_capi/test_misc.py

@@ -681,6 +681,20 @@ def test_heaptype_with_custom_metaclass(self):
  with self.assertRaisesRegex(TypeError, msg):
  t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew)
 
+ def test_heaptype_with_custom_metaclass_deprecation(self):
+ # gh-103968: a metaclass with custom tp_new is deprecated, but still
+ # allowed for functions that existed in 3.11
+ # (PyType_FromSpecWithBases is used here).
+ class Base(metaclass=_testcapi.HeapCTypeMetaclassCustomNew):
+ pass
+
+ with warnings_helper.check_warnings(
+ ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning),
+ ):
+ sub = _testcapi.make_type_with_base(Base)
+ self.assertTrue(issubclass(sub, Base))
+ self.assertIsInstance(sub, _testcapi.HeapCTypeMetaclassCustomNew)
+
  def test_multiple_inheritance_ctypes_with_weakref_or_dict(self):
 
  with self.assertRaises(TypeError):

Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst

@@ -0,0 +1,4 @@
+:c:func:`PyType_FromSpec` and its variants now allow creating classes whose
+metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is
+ignored. This behavior is deprecated and will be disallowed in 3.14+. The
+new :c:func:`PyType_FromMetaclass` already disallows it.

Modules/_testcapi/heaptype.c

@@ -22,7 +22,7 @@ static PyObject *pytype_fromspec_meta(PyObject* self, PyObject *meta)
  "_testcapi.HeapCTypeViaMetaclass",
  sizeof(PyObject),
  0,
- Py_TPFLAGS_DEFAULT,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
  HeapCTypeViaMetaclass_slots
  };
 
@@ -385,6 +385,19 @@ make_immutable_type_with_base(PyObject *self, PyObject *base)
  return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base);
 }
 
+static PyObject *
+make_type_with_base(PyObject *self, PyObject *base)
+{
+ assert(PyType_Check(base));
+ PyType_Spec ImmutableSubclass_spec = {
+ .name = "_testcapi.Subclass",
+ .basicsize = (int)((PyTypeObject*)base)->tp_basicsize,
+ .slots = empty_type_slots,
+ .flags = Py_TPFLAGS_DEFAULT,
+ };
+ return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base);
+}
+
 
 static PyMethodDef TestMethods[] = {
  {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O},
@@ -397,6 +410,7 @@ static PyMethodDef TestMethods[] = {
  test_from_spec_invalid_metatype_inheritance,
  METH_NOARGS},
  {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O},
+ {"make_type_with_base", make_type_with_base, METH_O},
  {NULL},
 };
 

Objects/typeobject.c

@@ -3830,9 +3830,10 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type)
  return 1;
 }
 
-PyObject *
-PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
- PyType_Spec *spec, PyObject *bases_in)
+static PyObject *
+_PyType_FromMetaclass_impl(
+ PyTypeObject *metaclass, PyObject *module,
+ PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new)
 {
  /* Invariant: A non-NULL value in one of these means this function holds
  * a strong reference or owns allocated memory.
@@ -4007,9 +4008,21 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
  goto finally;
  }
  if (metaclass->tp_new != PyType_Type.tp_new) {
- PyErr_SetString(PyExc_TypeError,
- "Metaclasses with custom tp_new are not supported.");
- goto finally;
+ if (_allow_tp_new) {
+ if (PyErr_WarnFormat(
+ PyExc_DeprecationWarning, 1,
+ "Using PyType_Spec with metaclasses that have custom "
+ "tp_new is deprecated and will no longer be allowed in "
+ "Python 3.14.") < 0) {
+ goto finally;
+ }
+ }
+ else {
+ PyErr_SetString(
+ PyExc_TypeError,
+ "Metaclasses with custom tp_new are not supported.");
+ goto finally;
+ }
  }
 
  /* Calculate best base, and check that all bases are type objects */
@@ -4196,22 +4209,29 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
  return (PyObject*)res;
 }
 
+PyObject *
+PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
+ PyType_Spec *spec, PyObject *bases_in)
+{
+ return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0);
+}
+
 PyObject *
 PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
 {
- return PyType_FromMetaclass(NULL, module, spec, bases);
+ return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1);
 }
 
 PyObject *
 PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 {
- return PyType_FromMetaclass(NULL, NULL, spec, bases);
+ return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1);
 }
 
 PyObject *
 PyType_FromSpec(PyType_Spec *spec)
 {
- return PyType_FromMetaclass(NULL, NULL, spec, NULL);
+ return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1);
 }
 
 PyObject *