GH-80789: Bundle ensurepip wheels at build time

.cirrus-DISABLED.yml

@@ -23,6 +23,9 @@ freebsd_task:
  pythoninfo_script:
  - cd build
  - make pythoninfo
+ # Download wheels for the venv step
+ bundle_ensurepip_script:
+ - cd build && make download-ensurepip-wheels
  test_script:
  - cd build
  # dtrace fails to build on FreeBSD - see gh-73263

.github/workflows/build.yml

@@ -435,6 +435,12 @@ jobs:
  - name: Remount sources writable for tests
  # some tests write to srcdir, lack of pyc files slows down testing
  run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+ # Download wheels for the venv step
+ - name: Download ensurepip wheels
+ env:
+ SSL_CERT_DIR: /etc/ssl/certs
+ run: make download-ensurepip-wheels
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
  - name: Setup directory envs for out-of-tree builds
  run: |
  echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV

.github/workflows/reusable-docs.yml

@@ -101,6 +101,9 @@ jobs:
  run: ./configure --with-pydebug
  - name: 'Build CPython'
  run: make -j4
+ # Download wheels for the venv step
+ - name: 'Download ensurepip wheels'
+ run: make download-ensurepip-wheels
  - name: 'Install build dependencies'
  run: make -C Doc/ PYTHON=../python venv
  # Use "xvfb-run" since some doctest tests open GUI windows

Doc/whatsnew/3.13.rst

@@ -907,6 +907,10 @@ Build Changes
 * Building CPython now requires a compiler with support for the C11 atomic
  library, GCC built-in atomic functions, or MSVC interlocked intrinsics.
 
+* Wheels for :mod:`ensurepip` are no longer bundled in the CPython source
+ tree. Distributors should bundle these as part of the build process by
+ running :file:`Tools/build/bundle_ensurepip_wheels.py`.
+
 
 C API Changes
 =============

Lib/ensurepip/__init__.py

@@ -11,8 +11,9 @@
 __all__ = ["version", "bootstrap"]
 _PACKAGE_NAMES = ('pip',)
 _PIP_VERSION = "23.2.1"
+_PIP_SHA_256 = "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be"
 _PROJECTS = [
- ("pip", _PIP_VERSION, "py3"),
+ ("pip", _PIP_VERSION, _PIP_SHA_256),
 ]
 
 # Packages bundled in ensurepip._bundled have wheel_name set.
@@ -62,8 +63,8 @@ def _get_packages():
  return _PACKAGES
 
  packages = {}
- for name, version, py_tag in _PROJECTS:
- wheel_name = f"{name}-{version}-{py_tag}-none-any.whl"
+ for name, version, _checksum in _PROJECTS:
+ wheel_name = f"{name}-{version}-py3-none-any.whl"
  packages[name] = _Package(version, wheel_name, None)
  if _WHEEL_PKG_DIR:
  dir_packages = _find_packages(_WHEEL_PKG_DIR)

Lib/ensurepip/_bundled/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.rst

Lib/ensurepip/_bundled/README.rst

@@ -0,0 +1,10 @@
+Bundling ensurepip wheels
+=========================
+
+``ensurepip`` expects wheels for ``pip`` to be located in this directory.
+These need to be 'bundled' by each distributor of Python,
+ordinarily by running the ``Tools/build/bundle_ensurepip_wheels.py`` script.
+
+This will download the version of ``pip`` specified by the
+``ensurepip._PIP_VERSION`` variable to this directory,
+and verify it against the stored SHA-256 checksum.

Lib/test/test_ensurepip.py

@@ -76,6 +76,12 @@ def setUp(self):
  patched_os.path = os.path
  self.os_environ = patched_os.environ = os.environ.copy()
 
+ # ensure the pip wheel exists
+ pip_filename = os.path.join(test.support.STDLIB_DIR, 'ensurepip', '_bundled',
+ f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl')
+ if not os.path.isfile(pip_filename):
+ open(pip_filename, "wb").close()
+
 
 class TestBootstrap(EnsurepipMixin, unittest.TestCase):
 

Lib/test/test_tools/test_bundle_ensurepip_wheels.py

@@ -0,0 +1,100 @@
+import sys
+import tempfile
+import unittest
+import unittest.mock
+import urllib.request
+from hashlib import sha256
+from io import BytesIO
+from pathlib import Path
+from random import randbytes
+from test import support, test_tools
+from test.support import captured_stderr
+
+import ensurepip
+
+test_tools.skip_if_missing('build')
+with test_tools.imports_under_tool('build'):
+ import bundle_ensurepip_wheels as bew
+
+
+# Disable fancy GitHub actions output during the tests
+@unittest.mock.patch.object(bew, 'GITHUB_ACTIONS', False)
+class TestBundle(unittest.TestCase):
+ contents = randbytes(512)
+ checksum = sha256(contents).hexdigest()
+ projects = [('pip', '1.2.3', checksum)]
+ pip_filename = "pip-1.2.3-py3-none-any.whl"
+
+ def test__wheel_url(self):
+ self.assertEqual(
+ bew._wheel_url('pip', '1.2.3'),
+ 'https://files.pythonhosted.org/packages/py3/p/pip/pip-1.2.3-py3-none-any.whl',
+ )
+
+ def test__get_projects(self):
+ self.assertListEqual(
+ bew._get_projects(),
+ [('pip', ensurepip._PIP_VERSION, ensurepip._PIP_SHA_256)],
+ )
+
+ def test__is_valid_wheel(self):
+ self.assertTrue(bew._is_valid_wheel(self.contents, checksum=self.checksum))
+
+ def test_cached_wheel(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ tmpdir = Path(tmpdir)
+ (tmpdir / self.pip_filename).write_bytes(self.contents)
+ with (
+ captured_stderr() as stderr,
+ unittest.mock.patch.object(bew, 'WHEEL_DIR', tmpdir),
+ unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+ exit_code = bew.download_wheels()
+ self.assertEqual(exit_code, 0)
+ stderr = stderr.getvalue()
+ self.assertIn("A valid 'pip' wheel already exists!", stderr)
+
+ def test_invalid_checksum(self):
+ class MockedHTTPSOpener:
+ @staticmethod
+ def open(url, data, timeout):
+ assert 'pip' in url
+ assert data is None # HTTP GET
+ # Intentionally corrupt the wheel:
+ return BytesIO(self.contents[:-1])
+
+ with (
+ captured_stderr() as stderr,
+ unittest.mock.patch.object(urllib.request, '_opener', None),
+ unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+ urllib.request.install_opener(MockedHTTPSOpener())
+ exit_code = bew.download_wheels()
+ self.assertEqual(exit_code, 1)
+ stderr = stderr.getvalue()
+ self.assertIn("Failed to validate checksum for", stderr)
+
+ def test_download_wheel(self):
+ class MockedHTTPSOpener:
+ @staticmethod
+ def open(url, data, timeout):
+ assert 'pip' in url
+ assert data is None # HTTP GET
+ return BytesIO(self.contents)
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ tmpdir = Path(tmpdir)
+ with (
+ captured_stderr() as stderr,
+ unittest.mock.patch.object(urllib.request, '_opener', None),
+ unittest.mock.patch.object(bew, 'WHEEL_DIR', tmpdir),
+ unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+ urllib.request.install_opener(MockedHTTPSOpener())
+ exit_code = bew.download_wheels()
+ self.assertEqual((tmpdir / self.pip_filename).read_bytes(), self.contents)
+ self.assertEqual(exit_code, 0)
+ stderr = stderr.getvalue()
+ self.assertIn("Downloading 'https://files.pythonhosted.org/packages/py3/p/pip/pip-1.2.3-py3-none-any.whl'",
+ stderr)
+ self.assertIn("Writing 'pip-1.2.3-py3-none-any.whl' to disk", stderr)

Makefile.pre.in

@@ -897,6 +897,14 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
 python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
 	@cp $(srcdir)/Tools/wasm/python.worker.js $@
 
+##########################################################################
+# Bundle wheels for ensurepip
+.PHONY: download-ensurepip-wheels
+download-ensurepip-wheels: $(BUILDPYTHON) platform
+	if test "x$(ENSUREPIP)" != "xno" ; then \
+ $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/bundle_ensurepip_wheels.py ; \
+	fi
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED)

Misc/NEWS.d/next/Library/2019-04-11-22-21-28.bpo-36608.HFoazc.rst

@@ -0,0 +1,2 @@
+Replaced vendored ``pip`` wheels for :mod:`ensurepip` with a new bundler script,
+:file:`Tools/build/bundle_ensurepip_wheels.py`, to be run by distributors.

Tools/build/bundle_ensurepip_wheels.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+"""
+Download wheels for 'ensurepip' packages from PyPI.
+
+When GitHub Actions executes the script, output is formatted accordingly.
+https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message
+"""
+
+import importlib.util
+import os
+import sys
+from hashlib import sha256
+from pathlib import Path
+from urllib.error import URLError
+from urllib.request import urlopen
+
+HOST = 'https://files.pythonhosted.org'
+ENSURE_PIP_ROOT = Path(__file__, "..", "..", "..", "Lib", "ensurepip").resolve()
+WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled"
+ENSURE_PIP_INIT = ENSURE_PIP_ROOT / "__init__.py"
+GITHUB_ACTIONS = "GITHUB_ACTIONS" in os.environ
+
+
+def print_notice(message: str) -> None:
+ if GITHUB_ACTIONS:
+ print(f"::notice::{message}", end="\n\n")
+ else:
+ print(message, file=sys.stderr)
+
+
+def print_error(message: str) -> None:
+ if GITHUB_ACTIONS:
+ print(f"::error::{message}", end="\n\n")
+ else:
+ print(message, file=sys.stderr)
+
+
+def download_wheels() -> int:
+ """Download wheels into bundle if they are not there yet."""
+
+ try:
+ projects = _get_projects()
+ except (AttributeError, TypeError):
+ print_error(f"Could not find '_PROJECTS' in {ENSURE_PIP_INIT}.")
+ return 1
+
+ errors = 0
+ for name, version, checksum in projects:
+ wheel_filename = f'{name}-{version}-py3-none-any.whl'
+ wheel_path = WHEEL_DIR / wheel_filename
+ if wheel_path.exists():
+ if _is_valid_wheel(wheel_path.read_bytes(), checksum=checksum):
+ print_notice(f"A valid '{name}' wheel already exists!")
+ continue
+ else:
+ print_error(f"An invalid '{name}' wheel exists.")
+ os.remove(wheel_path)
+
+ wheel_url = _wheel_url(name, version)
+ print_notice(f"Downloading {wheel_url!r}")
+ try:
+ with urlopen(wheel_url) as response:
+ whl = response.read()
+ except URLError as exc:
+ print_error(f"Failed to download {wheel_url!r}: {exc}")
+ errors = 1
+ continue
+ if not _is_valid_wheel(whl, checksum=checksum):
+ print_error(f"Failed to validate checksum for {wheel_url!r}!")
+ errors = 1
+ continue
+
+ print_notice(f"Writing {wheel_filename!r} to disk")
+ wheel_path.write_bytes(whl)
+
+ return errors
+
+
+def _get_projects() -> list[tuple[str, str, str]]:
+ spec = importlib.util.spec_from_file_location("ensurepip", ENSURE_PIP_INIT)
+ ensurepip = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(ensurepip)
+ return ensurepip._PROJECTS
+
+
+def _wheel_url(name: str, version: str, /) -> str:
+ # The following code was adapted from
+ # https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
+ #
+ # We rely on the fact that pip is, as a matter of policy, portable code.
+ # We can therefore guarantee that we'll always have the values:
+ # build_tag = ""
+ # python_tag = "py3"
+ # abi_tag = "none"
+ # platform_tag = "any"
+
+ # https://www.python.org/dev/peps/pep-0491/#file-name-convention
+ filename = f'{name}-{version}-py3-none-any.whl'
+ return f'{HOST}/packages/py3/{name[0]}/{name}/{filename}'
+
+
+def _is_valid_wheel(content: bytes, *, checksum: str) -> bool:
+ return checksum == sha256(content, usedforsecurity=False).hexdigest()
+
+
+if __name__ == '__main__':
+ raise SystemExit(download_wheels())