Fix threading-related deadlocks, and also add support for Python's built-in listing of libraries

CHANGES.md

@@ -34,6 +34,15 @@
 - Fix OpenBLAS detection for conda package on Windows
   https://github.com/joblib/threadpoolctl/pull/240
 
+- Fixed a deadlock on Linux when using threadpoolctl from multiple threads.
+  https://github.com/joblib/threadpoolctl/pull/243
+
+- Start using Python 3.14's built-in support for listing shared libraries.
+  https://github.com/joblib/threadpoolctl/pull/243
+
+- On Linux, start using /proc/self/maps for listing shared libraries.
+  https://github.com/joblib/threadpoolctl/pull/243
+
 3.6.0 (2025-03-13)
 ==================
 

tests/test_threadpoolctl.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+import ctypes
 import json
 import os
 import pytest
 import re
 import subprocess
 import sys
 from shutil import which
+from threading import Thread
 
 from threadpoolctl import threadpool_limits, threadpool_info
 from threadpoolctl import LibController, ThreadpoolController
@@ -952,3 +954,39 @@ def test_conda_blas_detection_after_import(module):
         if each["internal_api"] != "flexiblas"
     ]
     assert set(blas_names_from_threadpoolctl).issubset(blas_names_from_conda)
+
+
+def test_controller_parallelism_no_deadlocks():
+    """
+    Creating a controller in parallel to itself does not cause deadlocks.
+
+    Non-regression test for https://github.com/joblib/threadpoolctl/issues/239
+
+    Lacking the fixes from PR #243, this deadlocks on Conda environments, at
+    least, but possibly not on PyPI with Python from a Linux distro.
+    """
+    if sys.platform != "linux" or not hasattr(ctypes.PyDLL(None), "backtrace"):
+        pytest.skip("Testing glibc on Linux")
+
+    # Internally, backtrace() calls dl_iterate_phdr which can result in
+    # deadlocks if threadpoolctl is also using dl_iterate_phdr.
+    backtrace_gil = ctypes.PyDLL(None).backtrace
+    backtrace_gil.argtypes = [ctypes.c_void_p, ctypes.c_int]
+    backtrace_nogil = ctypes.CDLL(None).backtrace
+    backtrace_nogil.argtypes = [ctypes.c_void_p, ctypes.c_int]
+
+    def create_controllers():
+        buf = (ctypes.c_void_p * 20)()
+        for _ in range(100):
+            limiter = threadpool_limits()
+            backtrace_gil(buf, 20)
+            backtrace_nogil(buf, 20)
+
+    threads = []
+    for _ in range(os.cpu_count() * 4):
+        t = Thread(target=create_controllers)
+        threads.append(t)
+        t.start()
+
+    for t in threads:
+        t.join()

threadpoolctl.py

@@ -25,6 +25,12 @@
 from functools import lru_cache
 from contextlib import ContextDecorator
 
+try:
+    from ctypes.util import dllist
+except ImportError:
+    dllist = None
+
+
 __version__ = "3.7.0.dev0"
 __all__ = [
     "threadpool_limits",
@@ -1122,15 +1128,63 @@ def __len__(self):
 
     def _load_libraries(self):
         """Loop through loaded shared libraries and store the supported ones"""
-        if sys.platform == "darwin":
+        if sys.platform == "linux" and os.path.exists("/proc/self/maps"):
+            # On glibc, dl_iterate_phdr has an internal lock, and that plus
+            # calling back into Python and the need to (re)acquire the GIL
+            # results in deadlocks. To avoid that, use a Linux-specific
+            # mechanism that doesn't have these issues; since it's Linux, musl
+            # works fine too.
+            self._find_libraries_with_linux()
+        elif dllist is not None and sys.platform != "emscripten":
+            # On Python 3.14+, this functionality is built-in. Once Python 3.13
+            # is no longer supported by threadpoolctl, most of the equivalent
+            # threadpoolctl implementations can be removed.
+            #
+            # We don't use this on Linux since it uses dl_iterate_phdr
+            # internally and so might still have deadlock issues.
+            self._find_libraries_with_python()
+        elif sys.platform == "darwin":
             self._find_libraries_with_dyld()
         elif sys.platform == "win32":
             self._find_libraries_with_enum_process_module_ex()
         elif "pyodide" in sys.modules:
             self._find_libraries_pyodide()
         else:
+            # Non-Linux Unix platforms.
             self._find_libraries_with_dl_iterate_phdr()
 
+    def _find_libraries_with_linux(self):
+        """Loop through loaded libraries and return binders on supported ones
+
+        Uses a Linux-specific mechanism:
+        https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html
+        """
+        with open("/proc/self/maps") as f:
+            maps = f.read()
+        filepaths = set()
+        for line in maps.splitlines():
+            start_index = line.find("/")
+            if start_index == -1 or ".so" not in line:
+                continue
+            filepath = line[start_index:]
+            if os.path.exists(filepath):
+                filepaths.add(filepath)
+
+        for filepath in filepaths:
+            self._make_controller_from_path(filepath)
+
+    def _find_libraries_with_python(self):
+        """Loop through loaded libraries and return binders on supported ones
+
+        Uses Python's built-in support for this functionality.
+        """
+        assert dllist is not None
+        filepaths = dllist()
+        if filepaths and filepaths[0] in ("", sys.executable):
+            filepaths = filepaths[1:]
+        for filepath in filepaths:
+            self._make_controller_from_path(filepath)
+
     def _find_libraries_with_dl_iterate_phdr(self):
         """Loop through loaded libraries and return binders on supported ones