gh-100238: Use setuptools in peg-generator and reenable tests (#104798)

Author: lysnikolaou

Lib/test/support/__init__.py

@@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]):
  missing.
 
  """
-# TODO (PEP 632): alternate check without using distutils
-fromdistutilsimportccompiler, sysconfig, spawn, errors
+fromsetuptools._distutilsimportccompiler, sysconfig, spawn
+fromsetuptoolsimporterrors
+
 compiler=ccompiler.new_compiler()
 sysconfig.customize_compiler(compiler)
 ifcompiler.compiler_type=="msvc":
 # MSVC has no executables, so check whether initialization succeeds
 try:
 compiler.initialize()
-excepterrors.DistutilsPlatformError:
+excepterrors.PlatformError:
 return"msvc"
 fornameincompiler.executables:
 ifcmd_namesandnamenotincmd_names:
@@ -2270,6 +2271,42 @@ def requires_venv_with_pip():
 returnunittest.skipUnless(ctypes, 'venv: pip requires ctypes')
 
 
+# Context manager that creates a virtual environment, install setuptools and wheel in it
+# and returns the path to the venv directory and the path to the python executable
+@contextlib.contextmanager
+defsetup_venv_with_pip_setuptools_wheel(venv_dir):
+importsubprocess
+from .os_helperimporttemp_cwd
+
+withtemp_cwd() astemp_dir:
+# Create virtual environment to get setuptools
+cmd= [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
+ifverbose:
+print()
+print('Run:', ' '.join(cmd))
+subprocess.run(cmd, check=True)
+
+venv=os.path.join(temp_dir, venv_dir)
+
+# Get the Python executable of the venv
+python_exe=os.path.basename(sys.executable)
+ifsys.platform=='win32':
+python=os.path.join(venv, 'Scripts', python_exe)
+else:
+python=os.path.join(venv, 'bin', python_exe)
+
+cmd= [python, '-X', 'dev',
+'-m', 'pip', 'install',
+findfile('setuptools-67.6.1-py3-none-any.whl'),
+findfile('wheel-0.40.0-py3-none-any.whl')]
+ifverbose:
+print()
+print('Run:', ' '.join(cmd))
+subprocess.run(cmd, check=True)
+
+yieldpython
+
+
 # True if Python is built with the Py_DEBUG macro defined: if
 # Python is built in debug mode (./configure --with-pydebug).
 Py_DEBUG=hasattr(sys, 'gettotalrefcount')

Lib/test/test_cppext.py

@@ -35,39 +35,20 @@ def test_build_cpp03(self):
 # the test uses venv+pip: skip if it's not available
 @support.requires_venv_with_pip()
 defcheck_build(self, std_cpp03, extension_name):
-# Build in a temporary directory
-withos_helper.temp_cwd():
-self._check_build(std_cpp03, extension_name)
+venv_dir='env'
+withsupport.setup_venv_with_pip_setuptools_wheel(venv_dir) aspython_exe:
+self._check_build(std_cpp03, extension_name, python_exe)
 
-def_check_build(self, std_cpp03, extension_name):
+def_check_build(self, std_cpp03, extension_name, python_exe):
 pkg_dir='pkg'
 os.mkdir(pkg_dir)
 shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py"))
 
-venv_dir='env'
-verbose=support.verbose
-
-# Create virtual environment to get setuptools
-cmd= [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
-ifverbose:
-print()
-print('Run:', ' '.join(cmd))
-subprocess.run(cmd, check=True)
-
-# Get the Python executable of the venv
-python_exe='python'
-ifsys.executable.endswith('.exe'):
-python_exe+='.exe'
-ifMS_WINDOWS:
-python=os.path.join(venv_dir, 'Scripts', python_exe)
-else:
-python=os.path.join(venv_dir, 'bin', python_exe)
-
 defrun_cmd(operation, cmd):
 env=os.environ.copy()
 env['CPYTHON_TEST_CPP_STD'] ='c++03'ifstd_cpp03else'c++11'
 env['CPYTHON_TEST_EXT_NAME'] =extension_name
-ifverbose:
+ifsupport.verbose:
 print('Run:', ' '.join(cmd))
 subprocess.run(cmd, check=True, env=env)
 else:
@@ -81,29 +62,23 @@ def run_cmd(operation, cmd):
 self.fail(
 f"{operation} failed with exit code {proc.returncode}")
 
-cmd= [python, '-X', 'dev',
-'-m', 'pip', 'install',
-support.findfile('setuptools-67.6.1-py3-none-any.whl'),
-support.findfile('wheel-0.40.0-py3-none-any.whl')]
-run_cmd('Install build dependencies', cmd)
-
 # Build and install the C++ extension
-cmd= [python, '-X', 'dev',
+cmd= [python_exe, '-X', 'dev',
 '-m', 'pip', 'install', '--no-build-isolation',
 os.path.abspath(pkg_dir)]
 run_cmd('Install', cmd)
 
 # Do a reference run. Until we test that running python
 # doesn't leak references (gh-94755), run it so one can manually check
 # -X showrefcount results against this baseline.
-cmd= [python,
+cmd= [python_exe,
 '-X', 'dev',
 '-X', 'showrefcount',
 '-c', 'pass']
 run_cmd('Reference run', cmd)
 
 # Import the C++ extension
-cmd= [python,
+cmd= [python_exe,
 '-X', 'dev',
 '-X', 'showrefcount',
 '-c', f"import {extension_name}"]

Lib/test/test_peg_generator/__init__.py

@@ -3,9 +3,6 @@
 fromtestimportsupport
 fromtest.supportimportload_package_tests
 
-# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
-raiseunittest.SkipTest("distutils has been removed in Python 3.12")
-
 
 ifsupport.check_sanitizer(address=True, memory=True):
 # bpo-46633: Skip the test because it is too slow when Python is built

Lib/test/test_peg_generator/test_c_parser.py

@@ -1,3 +1,5 @@
+importcontextlib
+importsubprocess
 importsysconfig
 importtextwrap
 importunittest
@@ -8,7 +10,7 @@
 
 fromtestimporttest_tools
 fromtestimportsupport
-fromtest.supportimportos_helper
+fromtest.supportimportos_helper, import_helper
 fromtest.support.script_helperimportassert_python_ok
 
 _py_cflags_nodist=sysconfig.get_config_var("PY_CFLAGS_NODIST")
@@ -88,6 +90,16 @@ def setUpClass(cls):
 cls.library_dir=tempfile.mkdtemp(dir=cls.tmp_base)
 cls.addClassCleanup(shutil.rmtree, cls.library_dir)
 
+withcontextlib.ExitStack() asstack:
+python_exe=stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
+sitepackages=subprocess.check_output(
+ [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
+text=True,
+ ).strip()
+stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
+cls.addClassCleanup(stack.pop_all().close)
+
+@support.requires_venv_with_pip()
 defsetUp(self):
 self._backup_config_vars=dict(sysconfig._CONFIG_VARS)
 cmd=support.missing_compiler_executable()