gh-111964: Implement stop-the-world pauses

Include/cpython/pystate.h

@@ -102,7 +102,7 @@ struct _ts {
 #endif
  int _whence;
 
- /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC).
+ /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED).
  See Include/internal/pycore_pystate.h for more details. */
  int state;
 

Include/internal/pycore_ceval.h

@@ -205,6 +205,7 @@ void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame)
 #define _PY_CALLS_TO_DO_BIT 2
 #define _PY_ASYNC_EXCEPTION_BIT 3
 #define _PY_GC_SCHEDULED_BIT 4
+#define _PY_EVAL_PLEASE_STOP_BIT 5
 
 /* Reserve a few bits for future use */
 #define _PY_EVAL_EVENTS_BITS 8

Include/internal/pycore_interp.h

@@ -40,6 +40,22 @@ struct _Py_long_state {
  int max_str_digits;
 };
 
+// Support for stop-the-world events. This exists in both the PyRuntime struct
+// for global pauses and in each PyInterpreterState for per-interpreter pauses.
+struct _stoptheworld_state {
+ PyMutex mutex; // Serializes stop-the-world attempts.
+
+ // NOTE: The below fields are protected by HEAD_LOCK(runtime), not by the
+ // above mutex.
+ bool requested; // Set when a pause is requested.
+ bool world_stopped; // Set when the world is stopped.
+ bool is_global; // Set when contained in PyRuntime struct.
+
+ PyEvent stop_event; // Set when thread_countdown reaches zero.
+ Py_ssize_t thread_countdown; // Number of threads that must pause.
+
+ PyThreadState *requester; // Thread that requested the pause (may be NULL).
+};
 
 /* cross-interpreter data registry */
 
@@ -165,6 +181,7 @@ struct _is {
 
  struct _warnings_runtime_state warnings;
  struct atexit_state atexit;
+ struct _stoptheworld_state stoptheworld;
 
  struct _obmalloc_state obmalloc;
 

Include/internal/pycore_llist.h

@@ -37,8 +37,7 @@ struct llist_node {
 };
 
 // Get the struct containing a node.
-#define llist_data(node, type, member) \
- (type*)((char*)node - offsetof(type, member))
+#define llist_data(node, type, member) (_Py_CONTAINER_OF(node, type, member))
 
 // Iterate over a list.
 #define llist_for_each(node, head) \

Include/internal/pycore_pystate.h

@@ -19,23 +19,27 @@ extern "C" {
 // interpreter at the same time. Only the "bound" thread may perform the
 // transitions between "attached" and "detached" on its own PyThreadState.
 //
-// The "gc" state is used to implement stop-the-world pauses, such as for
-// cyclic garbage collection. It is only used in `--disable-gil` builds. It is
-// similar to the "detached" state, but only the thread performing a
-// stop-the-world pause may transition threads between the "detached" and "gc"
-// states. A thread trying to "attach" from the "gc" state will block until
-// it is transitioned back to "detached" when the stop-the-world pause is
-// complete.
+// The "suspended" state is used to implement stop-the-world pauses, such as
+// for cyclic garbage collection. It is only used in `--disable-gil` builds.
+// The "suspended" state is similar to the "detached" state in that in both
+// states the thread is not allowed to call most Python APIs. However, unlike
+// the "detached" state, a thread may not transition itself out from the
+// "suspended" state. Only the thread performing a stop-the-world pause may
+// transition a thread from the "suspended" state back to the "detached" state.
 //
 // State transition diagram:
 //
 // (bound thread) (stop-the-world thread)
-// [attached] <-> [detached] <-> [gc]
+// [attached] <-> [detached] <-> [suspended]
+// | ^
+// +---------------------------->---------------------------+
+// (bound thread)
 //
-// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`.
+// The (bound thread) and (stop-the-world thread) labels indicate which thread
+// is allowed to perform the transition.
 #define _Py_THREAD_DETACHED 0
 #define _Py_THREAD_ATTACHED 1
-#define _Py_THREAD_GC  2
+#define _Py_THREAD_SUSPENDED 2
 
 
 /* Check if the current thread is the main thread.
@@ -138,13 +142,36 @@ _PyThreadState_GET(void)
 //
 // High-level code should generally call PyEval_RestoreThread() instead, which
 // calls this function.
-void _PyThreadState_Attach(PyThreadState *tstate);
+extern void _PyThreadState_Attach(PyThreadState *tstate);
 
 // Detaches the current thread from the interpreter.
 //
 // High-level code should generally call PyEval_SaveThread() instead, which
 // calls this function.
-void _PyThreadState_Detach(PyThreadState *tstate);
+extern void _PyThreadState_Detach(PyThreadState *tstate);
+
+// Detaches the current thread to the "suspended" state if a stop-the-world
+// pause is in progress.
+//
+// If there is no stop-the-world pause in progress, then the thread switches
+// to the "detached" state.
+extern void _PyThreadState_Suspend(PyThreadState *tstate);
+
+// Perform a stop-the-world pause for all threads in the all interpreters.
+//
+// Threads in the "attached" state are paused and transitioned to the "GC"
+// state. Threads in the "detached" state switch to the "GC" state, preventing
+// them from reattaching until the stop-the-world pause is complete.
+//
+// NOTE: This is a no-op outside of Py_GIL_DISABLED builds.
+extern void _PyEval_StopTheWorldAll(_PyRuntimeState *runtime);
+extern void _PyEval_StartTheWorldAll(_PyRuntimeState *runtime);
+
+// Perform a stop-the-world pause for threads in the specified interpreter.
+//
+// NOTE: This is a no-op outside of Py_GIL_DISABLED builds.
+extern void _PyEval_StopTheWorld(PyInterpreterState *interp);
+extern void _PyEval_StartTheWorld(PyInterpreterState *interp);
 
 
 static inline void

Include/internal/pycore_runtime.h

@@ -227,6 +227,13 @@ typedef struct pyruntimestate {
  struct _faulthandler_runtime_state faulthandler;
  struct _tracemalloc_runtime_state tracemalloc;
 
+ // The rwmutex is used to prevent overlapping global and per-interpreter
+ // stop-the-world events. Global stop-the-world events lock the mutex
+ // exclusively (as a "writer"), while per-interpreter stop-the-world events
+ // lock it non-exclusively (as "readers").
+ _PyRWMutex stoptheworld_mutex;
+ struct _stoptheworld_state stoptheworld;
+
  PyPreConfig preconfig;
 
  // Audit values must be preserved when Py_Initialize()/Py_Finalize()

Include/internal/pycore_runtime_init.h

@@ -115,6 +115,9 @@ extern PyTypeObject _PyExc_MemoryError;
  }, \
  .faulthandler = _faulthandler_runtime_state_INIT, \
  .tracemalloc = _tracemalloc_runtime_state_INIT, \
+ .stoptheworld = { \
+ .is_global = 1, \
+ }, \
  .float_state = { \
  .float_format = _py_float_format_unknown, \
  .double_format = _py_float_format_unknown, \

Include/pymacro.h

@@ -160,6 +160,9 @@
  Py_FatalError("Unreachable C code path reached")
 #endif
 
+#define _Py_CONTAINER_OF(ptr, type, member) \
+ (type*)((char*)ptr - offsetof(type, member))
+
 // Prevent using an expression as a l-value.
 // For example, "int x; _Py_RVALUE(x) = 1;" fails with a compiler error.
 #define _Py_RVALUE(EXPR) ((void)0, (EXPR))

Python/ceval_gil.c

@@ -949,6 +949,15 @@ _Py_HandlePending(PyThreadState *tstate)
 {
  PyInterpreterState *interp = tstate->interp;
 
+ /* Stop-the-world */
+ if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_PLEASE_STOP_BIT)) {
+ _Py_set_eval_breaker_bit(interp, _PY_EVAL_PLEASE_STOP_BIT, 0);
+ _PyThreadState_Suspend(tstate);
+
+ /* The attach blocks until the stop-the-world event is complete. */
+ _PyThreadState_Attach(tstate);
+ }
+
  /* Pending signals */
  if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) {
  if (handle_signals(tstate) != 0) {