Skip to content

gh-104341: Call _PyEval_ReleaseLock() with NULL When Finalizing the Current Thread#105109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ericsnowcurrently merged 7 commits into from
Jun 1, 2023
Merged

gh-104341: Call _PyEval_ReleaseLock() with NULL When Finalizing the Current Thread #105109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5206f3d
Call _PyEval_ReleaseLock() with NULL when finalizing the current thread.
ericsnowcurrently May 30, 2023
db6d7ac
Temporarily adjust tstate->_status.cleared during interpreter_clear().
ericsnowcurrently May 30, 2023
a374f48
Disable the broken asserts.
ericsnowcurrently May 30, 2023
432db0c
Add a note about the race in drop_gil().
ericsnowcurrently May 31, 2023
1280e0d
Fix a typo.
ericsnowcurrently May 31, 2023
cc1f2fe
Clarify a comment.
ericsnowcurrently May 31, 2023
b64b299
minor comment typo fix.
gpshead Jun 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Include/internal/pycore_ceval.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
extern void _PyEval_FiniGIL(PyInterpreterState *interp);

extern void _PyEval_AcquireLock(PyThreadState *tstate);
extern void _PyEval_ReleaseLock(PyThreadState *tstate);
extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *);
extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);

extern void _PyEval_DeactivateOpCache(void);
Expand Down
30 changes: 26 additions & 4 deletions Python/ceval_gil.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -278,6 +278,15 @@ static void recreate_gil(struct _gil_runtime_state *gil)
static void
drop_gil(struct _ceval_state *ceval, PyThreadState *tstate)
{
/* If tstate is NULL, the caller is indicating that we're releasing
the GIL for the last time in this thread. This is particularly
relevant when the current thread state is finalizing or its
interpreter is finalizing (either may be in an inconsistent
state). In that case the current thread will definitely
never try to acquire the GIL again. */
// XXX It may be more correct to check tstate->_status.finalizing.
// XXX assert(tstate == NULL || !tstate->_status.cleared);

struct _gil_runtime_state *gil = ceval->gil;
if (!_Py_atomic_load_relaxed(&gil->locked)) {
Py_FatalError("drop_gil: GIL is not locked");
Expand All@@ -298,7 +307,15 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate)
MUTEX_UNLOCK(gil->mutex);

#ifdef FORCE_SWITCHING
if (_Py_atomic_load_relaxed(&ceval->gil_drop_request) && tstate != NULL) {
/* We check tstate first in case we might be releasing the GIL for
the last time in this thread. In that case there's a possible
race with tstate->interp getting deleted after gil->mutex is
unlocked and before the following code runs, leading to a crash.
We can use (tstate == NULL) to indicate the thread is done with
the GIL, and that's the only time we might delete the
interpreter, so checking tstate first prevents the crash.
See https://github.com/python/cpython/issues/104341. */
if (tstate != NULL && _Py_atomic_load_relaxed(&ceval->gil_drop_request)) {
Comment thread
gpshead marked this conversation as resolved.
MUTEX_LOCK(gil->switch_mutex);
/* Not switched yet => wait */
if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate)
Expand DownExpand Up@@ -350,6 +367,9 @@ take_gil(PyThreadState *tstate)
int err = errno;

assert(tstate != NULL);
/* We shouldn't be using a thread state that isn't viable any more. */
// XXX It may be more correct to check tstate->_status.finalizing.
// XXX assert(!tstate->_status.cleared);

if (tstate_must_exit(tstate)) {
/* bpo-39877: If Py_Finalize() has been called and tstate is not the
Expand DownExpand Up@@ -625,10 +645,12 @@ _PyEval_AcquireLock(PyThreadState *tstate)
}

void
_PyEval_ReleaseLock(PyThreadState *tstate)
_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate)

@ericsnowcurrentlyericsnowcurrentlyMay 30, 2023

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The other key part of this change is here, where we pass in the interpreter separately from the thread state, which allows tstate to be NULL.

{
_Py_EnsureTstateNotNULL(tstate);
struct _ceval_state *ceval = &tstate->interp->ceval;
/* If tstate is NULL then we do not expect the current thread
to acquire the GIL ever again. */
assert(tstate == NULL || tstate->interp == interp);
struct _ceval_state *ceval = &interp->ceval;
drop_gil(ceval, tstate);
}

Expand Down
2 changes: 1 addition & 1 deletion Python/pylifecycle.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2035,7 +2035,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
const PyConfig *src_config;
if (save_tstate != NULL) {
// XXX Might new_interpreter() have been called without the GIL held?
_PyEval_ReleaseLock(save_tstate);
_PyEval_ReleaseLock(save_tstate->interp, save_tstate);
src_config = _PyInterpreterState_GetConfig(save_tstate->interp);
}
else
Expand Down
21 changes: 18 additions & 3 deletions Python/pystate.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -822,6 +822,12 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
p = p->next;
HEAD_UNLOCK(runtime);
}
if (tstate->interp == interp) {
/* We fix tstate->_status below when we for sure aren't using it
(e.g. no longer need the GIL). */
// XXX Eliminate the need to do this.
tstate->_status.cleared = 0;
}

/* It is possible that any of the objects below have a finalizer
that runs Python code or otherwise relies on a thread state
Expand DownExpand Up@@ -886,6 +892,12 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
Py_CLEAR(interp->builtins);
Py_CLEAR(interp->interpreter_trampoline);

if (tstate->interp == interp) {
/* We are now safe to fix tstate->_status.cleared. */
// XXX Do this (much) earlier?
tstate->_status.cleared = 1;
}

for (int i=0; i < DICT_MAX_WATCHERS; i++) {
interp->dict_state.watchers[i] = NULL;
}
Expand DownExpand Up@@ -930,6 +942,7 @@ _PyInterpreterState_Clear(PyThreadState *tstate)
}


static inline void tstate_deactivate(PyThreadState *tstate);
static void zapthreads(PyInterpreterState *interp);

void
Expand All@@ -943,7 +956,9 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
PyThreadState *tcur = current_fast_get(runtime);
if (tcur != NULL && interp == tcur->interp) {
/* Unset current thread. After this, many C API calls become crashy. */
_PyThreadState_Swap(runtime, NULL);
current_fast_clear(runtime);
tstate_deactivate(tcur);
_PyEval_ReleaseLock(interp, NULL);
}

zapthreads(interp);
Expand DownExpand Up@@ -1567,7 +1582,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate)
_Py_EnsureTstateNotNULL(tstate);
tstate_delete_common(tstate);
current_fast_clear(tstate->interp->runtime);
_PyEval_ReleaseLock(tstate);
_PyEval_ReleaseLock(tstate->interp, NULL);
free_threadstate(tstate);
}

Expand DownExpand Up@@ -1907,7 +1922,7 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts)
{
PyThreadState *oldts = current_fast_get(runtime);
if (oldts != NULL) {
_PyEval_ReleaseLock(oldts);
_PyEval_ReleaseLock(oldts->interp, oldts);
}
_swap_thread_states(runtime, oldts, newts);
if (newts != NULL) {
Expand Down