Data race in `compile_template` in `sre.c`

[colesbury]

Bug report

Concurrent accesses to module_state->compile_template in the free threaded build

cpython/Modules/_sre/sre.c

Lines 1169 to 1177 in 1feaecc

	/* delegate to Python code */
	PyObject *func = module_state->compile_template;
	if (func == NULL) {
	func = PyImport_ImportModuleAttrString("re", "_compile_template");
	if (func == NULL) {
	return NULL;
	}
	Py_XSETREF(module_state->compile_template, func);
	}

Linked PRs

• gh-129983: fix data race in compile_template in sre.c #130015
• [3.13] gh-129983: fix data race in compile_template in sre.c #130038

[tom-pytel]

This fixes the issue. Let me know if you want me to send up. Or if you prefer it can be done with mutex (but uglier).

$ git diff main
diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
index 0d8d4843d33..64ef6b202e5 100644
--- a/Modules/_sre/sre.c
+++ b/Modules/_sre/sre.c
@@ -1167,13 +1167,21 @@ compile_template(_sremodulestate *module_state,
 PatternObject *pattern, PyObject *template)
 {
 /* delegate to Python code */
- PyObject *func = module_state->compile_template;
+ PyObject *func = FT_ATOMIC_LOAD_PTR_RELAXED(module_state->compile_template);
 if (func == NULL) {
 func = PyImport_ImportModuleAttrString("re", "_compile_template");
 if (func == NULL) {
 return NULL;
 }
+#ifdef Py_GIL_DISABLED
+ PyObject *other_func = NULL;
+ if (!_Py_atomic_compare_exchange_ptr(&module_state->compile_template, &other_func, func)) {
+ Py_DECREF(func);
+ func = other_func;
+ }
+#else
 Py_XSETREF(module_state->compile_template, func);
+#endif
 }

 PyObject *args[] = {(PyObject *)pattern, template};

Reproducer:

import re
import threading


def test(b):
 b.wait()
 re.sub(r"(\d+)", r"\1kg", "Weight: 50")


def check(funcs, *args):
 barrier = threading.Barrier(len(funcs))
 thrds = []

 for func in funcs:
 thrd = threading.Thread(target=func, args=(barrier, *args))

 thrds.append(thrd)
 thrd.start()

 for thrd in thrds:
 thrd.join()


if __name__ == "__main__":
 while True:
 check([test] * 40)

Also, while testing this I ran into this error exactly once and no more, though I don't think it has anything to do with this particular issue or fix:

ThreadSanitizer:DEADLYSIGNAL
==470981==ERROR: ThreadSanitizer: SEGV on unknown address 0x7f5cda2d0320 (pc 0x7f5d1447ed0f bp 0x00000000000d sp 0x7f5c221bf980 T471055)
==470981==The signal is caused by a WRITE memory access.
 #0 unsigned long long func_cas<unsigned long long>(unsigned long long volatile*, unsigned long long, unsigned long long) ../../../../src/libsanitizer/tsan/tsan_interface_atomic.cpp:97 (libtsan.so.0+0x7ed0f)
 #1 AtomicCAS<long long unsigned int> ../../../../src/libsanitizer/tsan/tsan_interface_atomic.cpp:422 (libtsan.so.0+0x7ed0f)
 #2 __tsan_atomic64_compare_exchange_strong ../../../../src/libsanitizer/tsan/tsan_interface_atomic.cpp:786 (libtsan.so.0+0x86ac2)
 #3 _Py_atomic_compare_exchange_ssize Include/cpython/pyatomic_gcc.h:130 (python+0x2235f2)
 #4 _Py_TryIncRefShared Include/internal/pycore_object.h:582 (python+0x2235f2)
 #5 _Py_TryIncrefCompare Include/internal/pycore_object.h:602 (python+0x2235f2)
 #6 _Py_TryXGetRef Include/internal/pycore_object.h:652 (python+0x22d0df)
 #7 _Py_dict_lookup_threadsafe Objects/dictobject.c:1543 (python+0x22d0df)
 #8 dict_subscript Objects/dictobject.c:3392 (python+0x22d65b)
 #9 PyObject_GetItem Objects/abstract.c:158 (python+0x14ca14)
 #10 _PyEval_EvalFrameDefault Python/generated_cases.c.h:62 (python+0x41b7c2)
 #11 _PyEval_EvalFrame Include/internal/pycore_ceval.h:116 (python+0x46f61d)
 #12 _PyEval_Vector Python/ceval.c:1725 (python+0x46f61d)
 #13 _PyFunction_Vectorcall Objects/call.c:413 (python+0x1838dc)
 #14 _PyObject_VectorcallTstate Include/internal/pycore_call.h:167 (python+0x18b5e3)
 #15 method_vectorcall Objects/classobject.c:72 (python+0x18b5e3)
 #16 _PyVectorcall_Call Objects/call.c:273 (python+0x187428)
 #17 _PyObject_Call Objects/call.c:348 (python+0x187956)
 #18 PyObject_Call Objects/call.c:373 (python+0x1879db)
 #19 thread_run Modules/_threadmodule.c:354 (python+0x6669ab)
 #20 pythread_wrapper Python/thread_pthread.h:242 (python+0x57bf0d)
 #21 __tsan_thread_start_func ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:959 (libtsan.so.0+0x2e5ff)
 #22 start_thread nptl/pthread_create.c:442 (libc.so.6+0x94ac2)
 #23 <null> <null> (libc.so.6+0x12684f)

ThreadSanitizer can not provide additional info.
SUMMARY: ThreadSanitizer: SEGV ../../../../src/libsanitizer/tsan/tsan_interface_atomic.cpp:97 in unsigned long long func_cas<unsigned long long>(unsigned long long volatile*, unsigned long long, unsigned long long)
==470981==ABORTING

[colesbury]

Thanks - pease send a PR!

[kumaraditya303]

I think the much simpler solution would be to initialize compile_template in module exec in this case sre_exec, then there will not be any thread safety issues as module exec is exclusively executed first before any code.

[tom-pytel]

I think the much simpler solution would be to initialize compile_template in module exec in this case sre_exec

The code in compile_template() is a lazy import because re imports _sre, which is why you can't initialize compile_template in sre_exec. You can however import re in sre_exec and access it in compile_template(), do you want this one?

$ git diff main
diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
index 0d8d4843d33..c2b1be957f5 100644
--- a/Modules/_sre/sre.c
+++ b/Modules/_sre/sre.c
@@ -376,7 +376,7 @@ typedef struct {
 PyTypeObject *Match_Type;
 PyTypeObject *Scanner_Type;
 PyTypeObject *Template_Type;
- PyObject *compile_template; // reference to re._compile_template
+ PyObject *re_module;
 } _sremodulestate;
 
 static _sremodulestate *
@@ -1167,17 +1167,13 @@ compile_template(_sremodulestate *module_state,
 PatternObject *pattern, PyObject *template)
 {
 /* delegate to Python code */
- PyObject *func = module_state->compile_template;
+ PyObject *func = PyObject_GetAttrString(module_state->re_module, "_compile_template");
 if (func == NULL) {
- func = PyImport_ImportModuleAttrString("re", "_compile_template");
- if (func == NULL) {
- return NULL;
- }
- Py_XSETREF(module_state->compile_template, func);
+ return NULL;
 }
-
 PyObject *args[] = {(PyObject *)pattern, template};
 PyObject *result = PyObject_Vectorcall(func, args, 2, NULL);
+ Py_DECREF(func);
 
 if (result == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) {
 /* If the replacement string is unhashable (e.g. bytearray),
@@ -3342,7 +3338,7 @@ sre_traverse(PyObject *module, visitproc visit, void *arg)
 Py_VISIT(state->Match_Type);
 Py_VISIT(state->Scanner_Type);
 Py_VISIT(state->Template_Type);
- Py_VISIT(state->compile_template);
+ Py_VISIT(state->re_module);
 
 return 0;
 }
@@ -3356,7 +3352,7 @@ sre_clear(PyObject *module)
 Py_CLEAR(state->Match_Type);
 Py_CLEAR(state->Scanner_Type);
 Py_CLEAR(state->Template_Type);
- Py_CLEAR(state->compile_template);
+ Py_CLEAR(state->re_module);
 
 return 0;
 }
@@ -3409,6 +3405,12 @@ sre_exec(PyObject *m)
 goto error;
 }
 
+ PyObject *re = PyImport_ImportModule("re");
+ if (re == NULL) {
+ goto error;

[kumaraditya303]

The code in compile_template() is a lazy import because re imports _sre, which is why you can't initialize compile_template in sre_exec. You can however import re in sre_exec and access it in compile_template(), do you want this one?

Ah, we had similar issue in asyncio as well with circular import, it was solved by delaying the import of C impl later, probably we can do something similar here like

cpython/Modules/_asynciomodule.c

Line 4435 in 247b50d

// Must be done after types are added to avoid a circular dependency

[colesbury]

Although I generally prefer initializing module state in the exec function, I'm hesitant to delay or reorder the imports here and would rather just use the atomics.

[kumaraditya303]

The following seems to fix it:

From e1ef31d9417fe9247603fdc66f2acc5cecfbb074 Mon Sep 17 00:00:00 2001
From: Kumar Aditya <kumaraditya@python.org>
Date: Tue, 11 Feb 2025 15:55:55 +0000
Subject: [PATCH] fix re

---
 Lib/re/__init__.py | 16 +++++++++-------
 Modules/_sre/sre.c | 14 +++++++-------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py
index 7e8abbf6ffe..9773c643065 100644
--- a/Lib/re/__init__.py
+++ b/Lib/re/__init__.py
@@ -123,8 +123,16 @@
 """
 
 import enum
-from . import _compiler, _parser
 import functools
+
+_MAXCACHE = 512
+@functools.lru_cache(_MAXCACHE)
+def _compile_template(pattern, repl):
+ # internal: compile replacement pattern
+ return _sre.template(pattern, _parser.parse_template(repl, pattern))
+
+
+from . import _compiler, _parser
 import _sre
 
 
@@ -323,7 +331,6 @@ def escape(pattern):
 # _cache uses the LRU policy which has better hit rate.
 _cache = {} # LRU
 _cache2 = {} # FIFO
-_MAXCACHE = 512
 _MAXCACHE2 = 256
 assert _MAXCACHE2 < _MAXCACHE
 
@@ -371,11 +378,6 @@ def _compile(pattern, flags):
 _cache2[key] = p
 return p
 
-@functools.lru_cache(_MAXCACHE)
-def _compile_template(pattern, repl):
- # internal: compile replacement pattern
- return _sre.template(pattern, _parser.parse_template(repl, pattern))
-
 # register myself for pickling
 
 import copyreg
diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
index 0d8d4843d33..a2b5f82d8b9 100644
--- a/Modules/_sre/sre.c
+++ b/Modules/_sre/sre.c
@@ -1168,13 +1168,7 @@ compile_template(_sremodulestate *module_state,
 {
 /* delegate to Python code */
 PyObject *func = module_state->compile_template;
- if (func == NULL) {
- func = PyImport_ImportModuleAttrString("re", "_compile_template");
- if (func == NULL) {
- return NULL;
- }
- Py_XSETREF(module_state->compile_template, func);
- }
+ assert(func != NULL);
 
 PyObject *args[] = {(PyObject *)pattern, template};
 PyObject *result = PyObject_Vectorcall(func, args, 2, NULL);
@@ -3409,6 +3403,12 @@ sre_exec(PyObject *m)
 goto error;
 }
 
+ state->compile_template = PyImport_ImportModuleAttrString("re", "_compile_template");
+
+ if (state->compile_template == NULL) {
+ goto error;
+ }
+
 return 0;
 

It seems that we need to just import _sre after defining _compile_template to avoid circular import.

[tom-pytel]

It seems that we need to just import _sre after defining _compile_template to avoid circular import.

A bit fragile?

[colesbury]

I'd be a lot more confident backporting the atomic change to 3.13 than reordering functions and imports in re.