GH-115776: Embed the values array into the object, for "normal" Python objects.

Include/cpython/pystats.h

@@ -77,12 +77,11 @@ typedef struct _object_stats {
  uint64_t frees;
  uint64_t to_freelist;
  uint64_t from_freelist;
- uint64_t new_values;
+ uint64_t inline_values;
  uint64_t dict_materialized_on_request;
  uint64_t dict_materialized_new_key;
  uint64_t dict_materialized_too_big;
  uint64_t dict_materialized_str_subclass;
- uint64_t dict_dematerialized;
  uint64_t type_cache_hits;
  uint64_t type_cache_misses;
  uint64_t type_cache_dunder_hits;

Include/internal/pycore_code.h

@@ -79,7 +79,10 @@ typedef struct {
 typedef struct {
  uint16_t counter;
  uint16_t type_version[2];
- uint16_t keys_version[2];
+ union {
+ uint16_t keys_version[2];
+ uint16_t dict_offset;
+ };
  uint16_t descr[4];
 } _PyLoadMethodCache;
 

Include/internal/pycore_dict.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #include "pycore_freelist.h" // _PyFreeListState
 #include "pycore_identifier.h" // _Py_Identifier
-#include "pycore_object.h" // PyDictOrValues
+#include "pycore_object.h" // PyManagedDictPointer
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +181,10 @@ struct _dictkeysobject {
  * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
  */
 struct _dictvalues {
+ uint8_t capacity;
+ uint8_t size;
+ uint8_t embedded;
+ uint8_t valid;
  PyObject *values[1];
 };
 
@@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
  size_t index = (size_t)1 << dk->dk_log2_index_bytes;
  return (&indices[index]);
 }
+
 static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
  assert(dk->dk_kind == DICT_KEYS_GENERAL);
  return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
 #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
 
-#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
-#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
-
 #ifdef Py_GIL_DISABLED
 #define DICT_NEXT_VERSION(INTERP) \
  (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
  return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
 }
 
-extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
-PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
+extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
+
 PyAPI_FUNC(PyObject *)_PyDict_FromItems(
  PyObject *const *keys, Py_ssize_t keys_offset,
  PyObject *const *values, Py_ssize_t values_offset,
  Py_ssize_t length);
 
+static inline uint8_t *
+get_insertion_order_array(PyDictValues *values)
+{
+ return (uint8_t *)&values->values[values->capacity];
+}
+
 static inline void
 _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
 {
  assert(ix < SHARED_KEYS_MAX_SIZE);
- uint8_t *size_ptr = ((uint8_t *)values)-2;
- int size = *size_ptr;
- assert(size+2 < DICT_VALUES_SIZE(values));
- size++;
- size_ptr[-size] = (uint8_t)ix;
- *size_ptr = size;
+ int size = values->size;
+ uint8_t *array = get_insertion_order_array(values);
+ assert(size < values->capacity);
+ assert(((uint8_t)ix) == ix);
+ array[size] = (uint8_t)ix;
+ values->size = size+1;
+}
+
+static inline size_t
+shared_keys_usable_size(PyDictKeysObject *keys)
+{
+#ifdef Py_GIL_DISABLED
+ // dk_usable will decrease for each instance that is created and each
+ // value that is added. dk_nentries will increase for each value that
+ // is added. We want to always return the right value or larger.
+ // We therefore increase dk_nentries first and we decrease dk_usable
+ // second, and conversely here we read dk_usable first and dk_entries
+ // second (to avoid the case where we read entries before the increment
+ // and read usable after the decrement)
+ return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
+ _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
+#else
+ return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
+#endif
 }
 
+static inline size_t
+_PyInlineValuesSize(PyTypeObject *tp)
+{
+ PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
+ assert(keys != NULL);
+ size_t size = shared_keys_usable_size(keys);
+ size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
+ assert(prefix_size < 256);
+ return prefix_size + (size + 1) * sizeof(PyObject *);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object.h

@@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
 {
  assert(op != NULL);
  Py_SET_TYPE(op, typeobj);
- if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
- Py_INCREF(typeobj);
- }
+ assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
+ Py_INCREF(typeobj);
  _Py_NewReference(op);
 }
 
@@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
 
-extern int _PyObject_InitializeDict(PyObject *obj);
-int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
+void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
 extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
  PyObject *name, PyObject *value);
 PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
 #endif
 
 typedef union {
- PyObject *dict;
- /* Use a char* to generate a warning if directly assigning a PyDictValues */
- char *values;
-} PyDictOrValues;
+ PyDictObject *dict;
+} PyManagedDictPointer;
 
-static inline PyDictOrValues *
-_PyObject_DictOrValuesPointer(PyObject *obj)
+static inline PyManagedDictPointer *
+_PyObject_ManagedDictPointer(PyObject *obj)
 {
  assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
-}
-
-static inline int
-_PyDictOrValues_IsValues(PyDictOrValues dorv)
-{
- return ((uintptr_t)dorv.values) & 1;
+ return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
 }
 
 static inline PyDictValues *
-_PyDictOrValues_GetValues(PyDictOrValues dorv)
-{
- assert(_PyDictOrValues_IsValues(dorv));
- return (PyDictValues *)(dorv.values + 1);
-}
-
-static inline PyObject *
-_PyDictOrValues_GetDict(PyDictOrValues dorv)
+_PyObject_InlineValues(PyObject *obj)
 {
- assert(!_PyDictOrValues_IsValues(dorv));
- return dorv.dict;
-}
-
-static inline void
-_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
-{
- ptr->values = ((char *)values) - 1;
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
+ return (PyDictValues *)((char *)obj + sizeof(PyObject));
 }
 
 extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
-extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
 extern int _PyObject_IsInstanceDictEmpty(PyObject *);
 
 // Export for 'math' shared extension

Include/object.h

@@ -629,13 +629,18 @@ given type object has a specified feature.
 /* Track types initialized using _PyStaticType_InitBuiltin(). */
 #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
 
+/* The values array is placed inline directly after the rest of
+ * the object. Implies Py_TPFLAGS_HAVE_GC.
+ */
+#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
+
 /* Placement of weakref pointers are managed by the VM, not by the type.
  * The VM will automatically set tp_weaklistoffset.
  */
 #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
 
 /* Placement of dict (and values) pointers are managed by the VM, not by the type.
- * The VM will automatically set tp_dictoffset.
+ * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
  */
 #define Py_TPFLAGS_MANAGED_DICT (1 << 4)
 

Lib/test/test_capi/test_mem.py

@@ -148,8 +148,8 @@ class C(): pass
  self.assertIn(b'MemoryError', out)
  *_, count = line.split(b' ')
  count = int(count)
- self.assertLessEqual(count, i*5)
- self.assertGreaterEqual(count, i*5-2)
+ self.assertLessEqual(count, i*10)
+ self.assertGreaterEqual(count, i*10-4)
 
 
 # Py_GIL_DISABLED requires mimalloc (not malloc)