GH-100240: Generic freelist, applied to ints

Include/internal/pycore_interp.h

@@ -18,6 +18,7 @@ extern "C" {
 #include "pycore_dict_state.h" // struct _Py_dict_state
 #include "pycore_exceptions.h" // struct _Py_exc_state
 #include "pycore_floatobject.h" // struct _Py_float_state
+#include "pycore_pymem.h" // free lists
 #include "pycore_function.h" // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h" // struct _Py_async_gen_state
 #include "pycore_gc.h" // struct _gc_runtime_state
@@ -50,6 +51,9 @@ struct _Py_long_state {
 
 /* interpreter state */
 
+#define SMALL_OBJECT_FREELIST_SIZE 1024
+#define INTERP_NUM_FREELISTS 30
+
 /* PyInterpreterState holds the global state for one of the runtime's
  interpreters. Typically the initial (main) interpreter is the only one.
 
@@ -178,6 +182,7 @@ struct _is {
  struct _Py_context_state context;
  struct _Py_exc_state exc_state;
 
+ _PyFreeList freelists[INTERP_NUM_FREELISTS];
  struct ast_state ast;
  struct types_state types;
  struct callable_cache callable_cache;
@@ -230,6 +235,36 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
 PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
 PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
 
+#if SIZEOF_VOID_P == 4
+#define LOG_BASE_2_OF_FREELIST_QUANTUM 3
+#elif SIZEOF_VOID_P == 8
+#define LOG_BASE_2_OF_FREELIST_QUANTUM 4
+#else
+#error "void pointer size not in (32, 64)"
+#endif
+
+#define FREELIST_QUANTUM (2*SIZEOF_VOID_P)
+#define SIZE_TO_FREELIST_SIZE_CLASS(size) (((size) + FREELIST_QUANTUM - 1) >> \
+ LOG_BASE_2_OF_FREELIST_QUANTUM)
+#define FREELIST_INDEX_TO_ALLOCATED_SIZE(idx) ((idx) * FREELIST_QUANTUM)
+
+static inline PyObject*
+_PyInterpreterState_FreelistAlloc(PyInterpreterState *interp, Py_ssize_t size) {
+ Py_ssize_t index = SIZE_TO_FREELIST_SIZE_CLASS(size);
+ assert(index >= 0 && index < INTERP_NUM_FREELISTS);
+ return _PyFreeList_Alloc(&interp->freelists[index]);
+}
+
+static inline void
+_PyInterpreterState_FreelistFree(PyInterpreterState * interp, PyObject *op, Py_ssize_t size) {
+ /* todo: assert the size is correct? */
+ Py_ssize_t index = SIZE_TO_FREELIST_SIZE_CLASS(size);
+ assert(index >= 0 && index < INTERP_NUM_FREELISTS);
+ _PyFreeList_Free(&interp->freelists[index], op);
+}
+
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_long.h

@@ -129,6 +129,8 @@ _PyLong_IsPositiveSingleDigit(PyObject* sub) {
  return ((size_t)signed_size) <= 1;
 }
 
+void _PyLong_Free(PyLongObject *op);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pymem.h

@@ -91,6 +91,80 @@ PyAPI_FUNC(int) _PyMem_GetAllocatorName(
 PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator);
 
 
+/* Free lists.
+ *
+ * Free lists have a pointer to their first entry and
+ * the amount of space available allowing fast checks
+ * for emptiness and fullness.
+ * When empty they are half filled and when full they are
+ * completely emptied. This helps the underlying allocator
+ * avoid fragmentation and helps performance.
+ */
+
+typedef struct _freelist {
+ void *ptr;
+ uint32_t space;
+ uint16_t size;
+ uint16_t capacity;
+#ifdef Py_STATS
+ int size_class;
+#endif
+} _PyFreeList;
+
+extern void *_PyFreeList_HalfFillAndAllocate(_PyFreeList *list);
+extern void _PyFreeList_FreeToFull(_PyFreeList *list, void *ptr);
+extern void _PyFreeList_Clear(_PyFreeList *list);
+extern void _PyFreeList_Disable(_PyFreeList *list);
+
+static inline void *
+_PyFreeList_Alloc(_PyFreeList *list) {
+#ifdef Py_STATS
+ if (_py_stats) _py_stats->freelist_stats[list->size_class].allocations++;
+#endif
+ if (list->ptr != NULL) {
+ void *result = list->ptr;
+ list->ptr = *((void **)result);
+ list->space++;
+ return result;
+ }
+#ifdef Py_STATS
+ if (_py_stats) _py_stats->freelist_stats[list->size_class].empty++;
+#endif
+ return _PyFreeList_HalfFillAndAllocate(list);
+}
+
+static inline void
+_PyFreeList_Free(_PyFreeList *list, void *ptr) {
+#ifdef Py_STATS
+ if (_py_stats) _py_stats->freelist_stats[list->size_class].frees++;
+#endif
+ if (list->space) {
+ *((void **)ptr) = list->ptr;
+ list->ptr = ptr;
+ list->space--;
+ return;
+ }
+#ifdef Py_STATS
+ if (_py_stats) _py_stats->freelist_stats[list->size_class].full++;
+#endif
+ _PyFreeList_FreeToFull(list, ptr);
+}
+
+static inline void
+_PyFreeList_Init(_PyFreeList *list, int size_class, int size, int capacity)
+{
+ list->ptr = NULL;
+ list->size = size;
+#ifdef Py_STATS
+ list->size_class = size_class;
+#endif
+#if WITH_FREELISTS
+ list->space = list->capacity = capacity;
+#else
+ _PyFreeList_Disable(list);
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

Include/pystats.h

@@ -48,6 +48,13 @@ typedef struct _call_stats {
  uint64_t eval_calls[EVAL_CALL_KINDS];
 } CallStats;
 
+typedef struct _generic_freelist_stats {
+ uint64_t allocations;
+ uint64_t frees;
+ uint64_t empty;
+ uint64_t full;
+} GenericFreelistStats;
+
 typedef struct _object_stats {
  uint64_t increfs;
  uint64_t decrefs;
@@ -78,6 +85,7 @@ typedef struct _stats {
  OpcodeStats opcode_stats[256];
  CallStats call_stats;
  ObjectStats object_stats;
+ GenericFreelistStats freelist_stats[30]; // INTERP_NUM_FREELISTS
 } PyStats;
 
 

Objects/longobject.c

@@ -6,6 +6,7 @@
 #include "pycore_bitutils.h" // _Py_popcount32()
 #include "pycore_initconfig.h" // _PyStatus_OK()
 #include "pycore_long.h" // _Py_SmallInts
+#include "pycore_pymem.h" // Free lists
 #include "pycore_object.h" // _PyObject_InitVar()
 #include "pycore_pystate.h" // _Py_IsMainInterpreter()
 #include "pycore_runtime.h" // _PY_NSMALLPOSINTS
@@ -46,7 +47,7 @@ static inline void
 _Py_DECREF_INT(PyLongObject *op)
 {
  assert(PyLong_CheckExact(op));
- _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_Free);
 }
 
 static inline int
@@ -152,16 +153,20 @@ _PyLong_New(Py_ssize_t size)
  "too many digits in integer");
  return NULL;
  }
- /* Fast operations for single digit integers (including zero)
- * assume that there is always at least one digit present. */
- Py_ssize_t ndigits = size ? size : 1;
- /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) +
- sizeof(digit)*size. Previous incarnations of this code used
- sizeof(PyVarObject) instead of the offsetof, but this risks being
- incorrect in the presence of padding between the PyVarObject header
- and the digits. */
- result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
- ndigits*sizeof(digit));
+ assert(size >= 0);
+ if (size <= 1) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ result = (PyLongObject *)_PyInterpreterState_FreelistAlloc(interp, sizeof(PyLongObject));
+ }
+ else {
+ /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) +
+ sizeof(digit)*size. Previous incarnations of this code used
+ sizeof(PyVarObject) instead of the offsetof, but this risks being
+ incorrect in the presence of padding between the PyVarObject header
+ and the digits. */
+ result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
+ size*sizeof(digit));
+ }
  if (!result) {
  PyErr_NoMemory();
  return NULL;
@@ -201,11 +206,11 @@ _PyLong_FromMedium(sdigit x)
 {
  assert(!IS_SMALL_INT(x));
  assert(is_medium_int(x));
- /* We could use a freelist here */
- PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ PyLongObject *v = (PyLongObject *)_PyInterpreterState_FreelistAlloc(
+ interp, sizeof(PyLongObject));
  if (v == NULL) {
- PyErr_NoMemory();
- return NULL;
+ return PyErr_NoMemory();
  }
  Py_ssize_t sign = x < 0 ? -1: 1;
  digit abs_x = x < 0 ? -x : x;
@@ -267,6 +272,19 @@ _PyLong_FromSTwoDigits(stwodigits x)
  return _PyLong_FromLarge(x);
 }
 
+void
+_PyLong_Free(PyLongObject *op)
+{
+ if (PyLong_CheckExact(op) && IS_MEDIUM_VALUE(op)) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _PyInterpreterState_FreelistFree(interp, (PyObject*)op, sizeof(PyLongObject));
+ }
+ else
+ {
+ Py_TYPE(op)->tp_free((PyObject *)op);
+ }
+}
+
 int
 _PyLong_AssignValue(PyObject **target, Py_ssize_t value)
 {
@@ -6286,7 +6304,7 @@ PyTypeObject PyLong_Type = {
  "int", /* tp_name */
  offsetof(PyLongObject, long_value.ob_digit), /* tp_basicsize */
  sizeof(digit), /* tp_itemsize */
-0,  /* tp_dealloc */
+(destructor)_PyLong_Free, /* tp_dealloc */
  0, /* tp_vectorcall_offset */
  0, /* tp_getattr */
  0, /* tp_setattr */

Objects/obmalloc.c

@@ -717,6 +717,64 @@ PyObject_Free(void *ptr)
 # define LIKELY(value) (value)
 #endif
 
+
+void *
+_PyFreeList_HalfFillAndAllocate(_PyFreeList *list)
+{
+ assert(list->ptr == NULL);
+ if (list->capacity < 4) {
+ return PyObject_Malloc(list->size);
+ }
+ uint32_t i = 0;
+ for (; i < list->space>>1; i++) {
+ void* ptr = PyObject_Malloc(list->size);
+ if (ptr == NULL) {
+ break;
+ }
+ *((void**)ptr) = list->ptr;
+ list->ptr = ptr;
+ }
+ if (i == 0) {
+ return NULL;
+ }
+ void *result = list->ptr;
+ list->ptr = *((void **)result);
+ list->space -= (i-1);
+ return result;
+}
+
+void
+_PyFreeList_Clear(_PyFreeList *list)
+{
+ int space = 0;
+ void *head = list->ptr;
+ while (head) {
+ void *next = *((void**)head);
+ PyObject_Free(head);
+ head = next;
+ space++;
+ }
+ list->ptr = NULL;
+ list->space += space;
+}
+
+void
+_PyFreeList_Disable(_PyFreeList *list)
+{
+ list->space = list->capacity = 0;
+}
+
+void
+_PyFreeList_FreeToFull(_PyFreeList *list, void *ptr)
+{
+ assert(list->space == 0);
+ PyObject_Free(ptr);
+ if (list->ptr == NULL) {
+ return;
+ }
+ _PyFreeList_Clear(list);
+}
+
 #ifdef WITH_PYMALLOC
 
 #ifdef WITH_VALGRIND