Allow __slots__ on classes with Py_TPFLAGS_ITEMS_AT_END

[jbradaric]

Bug report

If a base class is a PyVarObject, classes inheriting from that base cannot add __dict__ or __weakref__ through __slots__ declaration on the class. If __slots__ is not empty, the following exception is raised:
TypeError: nonempty __slots__ not supported for subtype of '...'

Assume that we have a PyVarObject foo.FooBase class (implementation below). Trying to inherit from the class and requesting weakref support doesn't work.

import foo
class WithWeakrefAndDict(foo.FooBase):
 __slots__ = ('__weakref__', '__dict__')

`foo` module implementation

#define PY_SSIZE_T_CLEAN
#include <Python.h>

static const Py_ssize_t N_EXTRA = 4;

typedef struct {
 PyObject_VAR_HEAD
} FooBase;

static PyObject **
FooBase_get_storage(PyObject *self)
{
 char *addr = (char *)self;
 return (PyObject **)(addr + Py_TYPE(self)->tp_basicsize);
}

static int
FooBase_traverse(PyObject *self, visitproc visit, void *arg)
{
 PyObject **storage = FooBase_get_storage(self);
 for (int i = 0; i < N_EXTRA; i++) {
 Py_VISIT(storage[i]);
 }
 Py_VISIT(Py_TYPE(self));
 return 0;
}

static int
FooBase_clear(PyObject *self)
{
 PyObject **storage = FooBase_get_storage(self);
 for (int i = 0; i < N_EXTRA; i++) {
 Py_CLEAR(storage[i]);
 }
 return 0;
}

static void
FooBase_dealloc(PyObject *self)
{
 PyTypeObject *tp = Py_TYPE(self);
 PyObject_GC_UnTrack(self);
 FooBase_clear(self);
 PyObject_GC_Del(self);
 if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE)
 Py_DECREF(tp);
}

static PyObject *
FooBase_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
 PyVarObject *obj = PyObject_GC_NewVar(PyVarObject, type, N_EXTRA);
 if (obj == NULL)
 return NULL;
 PyObject **storage = FooBase_get_storage((PyObject *)obj);
 for (int i = 0; i < Py_SIZE(obj); i++)
 storage[i] = NULL;

 PyObject_GC_Track(obj);
 return (PyObject *)obj;
}

static PyObject *
FooBase_get_extra(PyObject *self, PyObject *args)
{
 Py_ssize_t idx;
 if (!PyArg_ParseTuple(args, "n", &idx))
 return NULL;
 if (idx < 0 || idx >= N_EXTRA) {
 PyErr_Format(PyExc_ValueError, "idx must be >= 0 and < %zd", N_EXTRA);
 return NULL;
 }

 PyObject **storage = FooBase_get_storage(self);
 PyObject *value = storage[idx];
 if (!value) {
 Py_RETURN_NONE;
 } else {
 Py_INCREF(value);
 return value;
 }
}

static PyObject *
FooBase_set_extra(PyObject *self, PyObject *args)
{
 Py_ssize_t idx;
 PyObject *value;
 if (!PyArg_ParseTuple(args, "nO", &idx, &value))
 return NULL;
 if (idx < 0 || idx >= N_EXTRA) {
 PyErr_Format(PyExc_ValueError, "idx must be >= 0 and < %zd", N_EXTRA);
 return NULL;
 }

 PyObject **storage = FooBase_get_storage(self);
 Py_CLEAR(storage[idx]);
 Py_INCREF(value);
 storage[idx] = value;

 Py_RETURN_NONE;
}

static PyMethodDef FooBase_methods[] = {
 {"get_extra", FooBase_get_extra, METH_VARARGS, NULL},
 {"set_extra", FooBase_set_extra, METH_VARARGS, NULL},
 {NULL}
};

static PyTypeObject FooBase_Type = {
 PyVarObject_HEAD_INIT(NULL, 0)
 "foo.FooBase",
 sizeof(FooBase),
 .tp_dealloc = (destructor)FooBase_dealloc,
 .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
 .tp_traverse = FooBase_traverse,
 .tp_clear = FooBase_clear,
 .tp_methods = FooBase_methods,
 .tp_new = FooBase_new,
 .tp_itemsize = sizeof(PyObject *)
};

static struct PyModuleDef foo_module = {
 PyModuleDef_HEAD_INIT,
 "foo",
 NULL,
 -1,
};

PyMODINIT_FUNC
PyInit_foo(void)
{
 PyObject *m = PyModule_Create(&foo_module);
 if (!m)
 return NULL;
 if (PyType_Ready(&FooBase_Type) < 0)
 return NULL;
 Py_INCREF(&FooBase_Type);
 PyModule_AddObject(m, "FooBase", (PyObject *)&FooBase_Type);
 return m;
}

Your environment

• CPython versions tested on: 3.11.3
• Operating system and architecture: Linux, x86_64

CC: @encukou

Linked PRs

• gh-103740: Improve support of __slots__ for variable-length classes #141636
• gh-103740: Support any slots in the type subclasses #141764

[ronaldoussoren]

This is a duplicate of #61497. The isn't isn't so much that these particular slots aren't supported, but that no slots are supported at all var these objects.

[encukou]

Let's use this issue for enabling this with Py_TPFLAGS_ITEMS_AT_END specifically. I plan to look into it for 3.13.

[serhiy-storchaka]

This particular issue (support __dict__ and __weakref__) can be solved relatively easy. Currently variable length types can have either both __dict__ and __weakref__, or none of them, so it is possible.

But I think that we should support also general slots, there are use cases for this. So this issue can be a part of #41779.

[serhiy-storchaka]

#141636 implements two features:

• Support __dict__ and __weakref__ slots for any class.
• Support any __slots__ for subclasses of type and classes with Py_TPFLAGS_ITEMS_AT_END.

If you with, I can split it on two PRs, if this will not slow down review.

My next step is to support any __slots__ for subclasses of tuple. And then we can plan support any variable-length types (it is not easy).