Skip to content

gh-121103: Put free-threaded libraries in lib/python3.14t#121293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
colesbury merged 5 commits into from
Jul 11, 2024
Merged

gh-121103: Put free-threaded libraries in lib/python3.14t#121293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
03772aa
gh-121103: Put free-threaded libraries in `lib/python3.14t`
colesbury Jul 1, 2024
ed2dde7
Fix blurb
colesbury Jul 2, 2024
6b46646
Make ABI logic in sysconfig a bit more consistent with everything else.
colesbury Jul 2, 2024
d72fb70
Use tab in Makefile.pre.in
colesbury Jul 11, 2024
432701c
Merge branch 'main' into gh-121103-libdir
colesbury Jul 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions Lib/site.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -312,6 +312,10 @@ def joinuser(*args):
# Same to sysconfig.get_path('purelib', os.name+'_user')
def _get_path(userbase):
version = sys.version_info
if hasattr(sys, 'abiflags') and 't' in sys.abiflags:
abi_thread = 't'
else:
abi_thread = ''

implementation = _get_implementation()
implementation_lower = implementation.lower()
Expand All@@ -322,7 +326,7 @@ def _get_path(userbase):
if sys.platform == 'darwin' and sys._framework:
return f'{userbase}/lib/{implementation_lower}/site-packages'

return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages'


def getuserbase():
Expand DownExpand Up@@ -390,14 +394,18 @@ def getsitepackages(prefixes=None):

implementation = _get_implementation().lower()
ver = sys.version_info
if hasattr(sys, 'abiflags') and 't' in sys.abiflags:
abi_thread = 't'
else:
abi_thread = ''
if os.sep == '/':
libdirs = [sys.platlibdir]
if sys.platlibdir != "lib":
libdirs.append("lib")

for libdir in libdirs:
path = os.path.join(prefix, libdir,
f"{implementation}{ver[0]}.{ver[1]}",
f"{implementation}{ver[0]}.{ver[1]}{abi_thread}",
Comment on lines +397 to +408

@AlexWaygoodAlexWaygoodAug 6, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@colesbury -- would you be able to update the docs for the site module to reflect these changes to how Python finds the site-packages directory?

(The context is I'm currently working on reimplementing Python's logic here for Astral's work-in-progress type checker. I was reading through the site.py source code, and it surprised me to find logic here that wasn't documented!)

@colesburycolesburyAug 6, 2024

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the pointer. I put up a PR to update the docs: #122737

"site-packages")
sitepackages.append(path)
else:
Expand Down
33 changes: 20 additions & 13 deletions Lib/sysconfig/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,10 +27,10 @@

_INSTALL_SCHEMES = {
'posix_prefix': {
'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}',
'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}',
'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages',
'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'include':
'{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
'platinclude':
Expand DownExpand Up@@ -77,10 +77,10 @@
# Downstream distributors who patch posix_prefix/nt scheme are encouraged to
# leave the following schemes unchanged
'posix_venv': {
'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}',
'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}',
'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages',
'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'include':
'{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
'platinclude':
Expand DownExpand Up@@ -148,11 +148,11 @@ def joinuser(*args):
'data': '{userbase}',
},
'posix_user': {
'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}',
'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}',
'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
'include': '{userbase}/include/{implementation_lower}{py_version_short}',
'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}',
'scripts': '{userbase}/bin',
'data': '{userbase}',
},
Expand DownExpand Up@@ -487,6 +487,9 @@ def _init_config_vars():
# the init-function.
_CONFIG_VARS['userbase'] = _getuserbase()

# e.g., 't' for free-threaded or '' for default build
_CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else ''

# Always convert srcdir to an absolute path
srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE)
if os.name == 'posix':
Expand DownExpand Up@@ -655,6 +658,10 @@ def get_python_version():
return _PY_VERSION_SHORT


def _get_python_version_abi():
return _PY_VERSION_SHORT + get_config_var("abi_thread")


def expand_makefile_vars(s, vars):
"""Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
'string' according to 'vars' (a dictionary mapping variable names to
Expand Down
14 changes: 8 additions & 6 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,6 +48,8 @@
INIT_LOOPS = 4
MAX_HASH_SEED = 4294967295

ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else ''


# If we are running from a build dir, but the stdlib has been installed,
# some tests need to expect different results.
Expand DownExpand Up@@ -1285,11 +1287,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None):
ver = sys.version_info
return [
os.path.join(prefix, sys.platlibdir,
f'python{ver.major}{ver.minor}.zip'),
f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'),
os.path.join(prefix, sys.platlibdir,
f'python{ver.major}.{ver.minor}'),
f'python{ver.major}.{ver.minor}{ABI_THREAD}'),
os.path.join(exec_prefix, sys.platlibdir,
f'python{ver.major}.{ver.minor}', 'lib-dynload'),
f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'),
]

@contextlib.contextmanager
Expand DownExpand Up@@ -1343,7 +1345,7 @@ def test_init_setpythonhome(self):
expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
else:
version = f'{sys.version_info.major}.{sys.version_info.minor}'
stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}')
expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)

config = {
Expand DownExpand Up@@ -1384,7 +1386,7 @@ def test_init_is_python_build_with_home(self):
expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
else:
version = f'{sys.version_info.major}.{sys.version_info.minor}'
stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}')
expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)

config = {
Expand DownExpand Up@@ -1515,7 +1517,7 @@ def test_init_pyvenv_cfg(self):
if not MS_WINDOWS:
lib_dynload = os.path.join(pyvenv_home,
sys.platlibdir,
f'python{ver.major}.{ver.minor}',
f'python{ver.major}.{ver.minor}{ABI_THREAD}',
'lib-dynload')
os.makedirs(lib_dynload)
else:
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_getpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -844,6 +844,7 @@ def test_explicitly_set_stdlib_dir(self):
PYDEBUGEXT="",
VERSION_MAJOR=9, # fixed version number for ease
VERSION_MINOR=8, # of testing
ABI_THREAD="",
PYWINVER=None,
EXE_SUFFIX=None,

Expand Down
4 changes: 2 additions & 2 deletions Lib/test/test_site.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -328,13 +328,13 @@ def test_getsitepackages(self):
if sys.platlibdir != "lib":
self.assertEqual(len(dirs), 2)
wanted = os.path.join('xoxo', sys.platlibdir,
'python%d.%d' % sys.version_info[:2],
f'python{sysconfig._get_python_version_abi()}',
'site-packages')
self.assertEqual(dirs[0], wanted)
else:
self.assertEqual(len(dirs), 1)
wanted = os.path.join('xoxo', 'lib',
'python%d.%d' % sys.version_info[:2],
f'python{sysconfig._get_python_version_abi()}',
'site-packages')
self.assertEqual(dirs[-1], wanted)
else:
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_sysconfig.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -157,7 +157,7 @@ def test_posix_venv_scheme(self):
binpath = 'bin'
incpath = 'include'
libpath = os.path.join('lib',
'python%d.%d' % sys.version_info[:2],
f'python{sysconfig._get_python_version_abi()}',
'site-packages')

# Resolve the paths in an imaginary venv/ directory
Expand Down
5 changes: 3 additions & 2 deletions Lib/test/test_venv.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -75,7 +75,7 @@ def setUp(self):
self.include = 'Include'
else:
self.bindir = 'bin'
self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}')
self.include = 'include'
executable = sys._base_executable
self.exe = os.path.split(executable)[-1]
Expand DownExpand Up@@ -593,7 +593,8 @@ def test_zippath_from_non_installed_posix(self):
libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1])
os.makedirs(libdir)
landmark = os.path.join(libdir, "os.py")
stdlib_zip = "python%d%d.zip" % sys.version_info[:2]
abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""
stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}"
zip_landmark = os.path.join(non_installed_dir,
platlibdir,
stdlib_zip)
Expand Down
3 changes: 2 additions & 1 deletion Makefile.pre.in
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ AR= @AR@
READELF= @READELF@
SOABI= @SOABI@
ABIFLAGS= @ABIFLAGS@
ABI_THREAD= @ABI_THREAD@
LDVERSION= @LDVERSION@
MODULE_LDFLAGS=@MODULE_LDFLAGS@
GITVERSION= @GITVERSION@
Expand DownExpand Up@@ -158,7 +159,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@

# Detailed destination directories
BINLIBDEST= @BINLIBDEST@
LIBDEST= $(SCRIPTDIR)/python$(VERSION)
LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD)
INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION)
CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION)

Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
On POSIX systems, excluding macOS framework installs, the lib directory
for the free-threaded build now includes a "t" suffix to avoid conflicts
with a co-located default build installation.
5 changes: 5 additions & 0 deletions Modules/getpath.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -951,6 +951,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
!wchar_to_dict(dict, "executable_dir", NULL) ||
!wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) ||
!funcs_to_dict(dict, config->pathconfig_warnings) ||
#ifdef Py_GIL_DISABLED
!decode_to_dict(dict, "ABI_THREAD", "t") ||
#else
!decode_to_dict(dict, "ABI_THREAD", "") ||
#endif
#ifndef MS_WINDOWS
PyDict_SetItemString(dict, "winreg", Py_None) < 0 ||
#endif
Expand Down
8 changes: 5 additions & 3 deletions Modules/getpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@
# EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar
# VERSION_MAJOR -- [in] sys.version_info.major
# VERSION_MINOR -- [in] sys.version_info.minor
# ABI_THREAD -- [in] either 't' for free-threaded builds or ''
# PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32)

# ** Values read from the environment **
Expand DownExpand Up@@ -172,17 +173,18 @@
# ******************************************************************************

platlibdir = config.get('platlibdir') or PLATLIBDIR
ABI_THREAD = ABI_THREAD or ''

if os_name == 'posix' or os_name == 'darwin':
BUILDDIR_TXT = 'pybuilddir.txt'
BUILD_LANDMARK = 'Modules/Setup.local'
DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}'
STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}'
STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}'
STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc']
PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload'
PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload'
BUILDSTDLIB_LANDMARKS = ['Lib/os.py']
VENV_LANDMARK = 'pyvenv.cfg'
ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip'
ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip'
DELIM = ':'
SEP = '/'

Expand Down
14 changes: 9 additions & 5 deletions configure
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 8 additions & 5 deletions configure.ac
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1739,7 +1739,9 @@ fi

# For calculating the .so ABI tag.
AC_SUBST([ABIFLAGS])
AC_SUBST([ABI_THREAD])
ABIFLAGS=""
ABI_THREAD=""

# Check for --disable-gil
# --disable-gil
Expand All@@ -1756,6 +1758,7 @@ then
[Define if you want to disable the GIL])
# Add "t" for "threaded"
ABIFLAGS="${ABIFLAGS}t"
ABI_THREAD="t"
fi

# Check for --with-pydebug
Expand DownExpand Up@@ -6234,11 +6237,11 @@ fi


AC_SUBST([BINLIBDEST])
BINLIBDEST='$(LIBDIR)/python$(VERSION)'
BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)'


# Check for --with-platlibdir
# /usr/$LIDIRNAME/python$VERSION
# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD)
AC_SUBST([PLATLIBDIR])
PLATLIBDIR="lib"
AC_MSG_CHECKING([for --with-platlibdir])
Expand All@@ -6257,7 +6260,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no
then
AC_MSG_RESULT([yes])
PLATLIBDIR="$withval"
BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)'
BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)'
else
AC_MSG_RESULT([no])
fi],
Expand All@@ -6267,9 +6270,9 @@ fi],
dnl define LIBPL after ABIFLAGS and LDVERSION is defined.
AC_SUBST([PY_ENABLE_SHARED])
if test x$PLATFORM_TRIPLET = x; then
LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}"
LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}"
else
LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
fi
AC_SUBST([LIBPL])

Expand Down