diff --git a/CHANGES.md b/CHANGES.md index c47d132b..b0b5e3bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,15 @@ - Fix OpenBLAS detection for conda package on Windows https://github.com/joblib/threadpoolctl/pull/240 +- Fixed a deadlock on Linux when using threadpoolctl from multiple threads. + https://github.com/joblib/threadpoolctl/pull/243 + +- Start using Python 3.14's built-in support for listing shared libraries. + https://github.com/joblib/threadpoolctl/pull/243 + +- On Linux, start using /proc/self/maps for listing shared libraries. + https://github.com/joblib/threadpoolctl/pull/243 + 3.6.0 (2025-03-13) ================== diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index adf83d62..652f8ae0 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ctypes import json import os import pytest @@ -7,6 +8,7 @@ import subprocess import sys from shutil import which +from threading import Thread from threadpoolctl import threadpool_limits, threadpool_info from threadpoolctl import LibController, ThreadpoolController @@ -952,3 +954,39 @@ def test_conda_blas_detection_after_import(module): if each["internal_api"] != "flexiblas" ] assert set(blas_names_from_threadpoolctl).issubset(blas_names_from_conda) + + +def test_controller_parallelism_no_deadlocks(): + """ + Creating a controller in parallel to itself does not cause deadlocks. + + Non-regression test for https://github.com/joblib/threadpoolctl/issues/239 + + Lacking the fixes from PR #243, this deadlocks on Conda environments, at + least, but possibly not on PyPI with Python from a Linux distro. + """ + if sys.platform != "linux" or not hasattr(ctypes.PyDLL(None), "backtrace"): + pytest.skip("Testing glibc on Linux") + + # Internally, backtrace() calls dl_iterate_phdr which can result in + # deadlocks if threadpoolctl is also using dl_iterate_phdr. + backtrace_gil = ctypes.PyDLL(None).backtrace + backtrace_gil.argtypes = [ctypes.c_void_p, ctypes.c_int] + backtrace_nogil = ctypes.CDLL(None).backtrace + backtrace_nogil.argtypes = [ctypes.c_void_p, ctypes.c_int] + + def create_controllers(): + buf = (ctypes.c_void_p * 20)() + for _ in range(100): + limiter = threadpool_limits() + backtrace_gil(buf, 20) + backtrace_nogil(buf, 20) + + threads = [] + for _ in range(os.cpu_count() * 4): + t = Thread(target=create_controllers) + threads.append(t) + t.start() + + for t in threads: + t.join() diff --git a/threadpoolctl.py b/threadpoolctl.py index 15146265..3faf26e1 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -25,6 +25,12 @@ from functools import lru_cache from contextlib import ContextDecorator +try: + from ctypes.util import dllist +except ImportError: + dllist = None + + __version__ = "3.7.0.dev0" __all__ = [ "threadpool_limits", @@ -1122,15 +1128,63 @@ def __len__(self): def _load_libraries(self): """Loop through loaded shared libraries and store the supported ones""" - if sys.platform == "darwin": + if sys.platform == "linux" and os.path.exists("/proc/self/maps"): + # On glibc, dl_iterate_phdr has an internal lock, and that plus + # calling back into Python and the need to (re)acquire the GIL + # results in deadlocks. To avoid that, use a Linux-specific + # mechanism that doesn't have these issues; since it's Linux, musl + # works fine too. + self._find_libraries_with_linux() + elif dllist is not None and sys.platform != "emscripten": + # On Python 3.14+, this functionality is built-in. Once Python 3.13 + # is no longer supported by threadpoolctl, most of the equivalent + # threadpoolctl implementations can be removed. + # + # We don't use this on Linux since it uses dl_iterate_phdr + # internally and so might still have deadlock issues. + self._find_libraries_with_python() + elif sys.platform == "darwin": self._find_libraries_with_dyld() elif sys.platform == "win32": self._find_libraries_with_enum_process_module_ex() elif "pyodide" in sys.modules: self._find_libraries_pyodide() else: + # Non-Linux Unix platforms. self._find_libraries_with_dl_iterate_phdr() + def _find_libraries_with_linux(self): + """Loop through loaded libraries and return binders on supported ones + + Uses a Linux-specific mechanism: + https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html + """ + with open("/proc/self/maps") as f: + maps = f.read() + filepaths = set() + for line in maps.splitlines(): + start_index = line.find("/") + if start_index == -1 or ".so" not in line: + continue + filepath = line[start_index:] + if os.path.exists(filepath): + filepaths.add(filepath) + + for filepath in filepaths: + self._make_controller_from_path(filepath) + + def _find_libraries_with_python(self): + """Loop through loaded libraries and return binders on supported ones + + Uses Python's built-in support for this functionality. + """ + assert dllist is not None + filepaths = dllist() + if filepaths and filepaths[0] in ("", sys.executable): + filepaths = filepaths[1:] + for filepath in filepaths: + self._make_controller_from_path(filepath) + def _find_libraries_with_dl_iterate_phdr(self): """Loop through loaded libraries and return binders on supported ones