bpo-45020: Freeze the modules imported during startup.

.gitattributes

@@ -47,6 +47,7 @@ Objects/clinic/*.h linguist-generated=true
 PC/clinic/*.h linguist-generated=true
 Python/clinic/*.h linguist-generated=true
 Python/frozen_modules/*.h linguist-generated=true
+Python/frozen_modules/MANIFEST linguist-generated=true
 Include/internal/pycore_ast.h linguist-generated=true
 Python/Python-ast.c linguist-generated=true
 Include/opcode.h linguist-generated=true

.gitignore

@@ -121,6 +121,13 @@ Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
 
+# TODO: Once we auto-regen frozem modules for Windows builds
+# we can drop the .h files from the repo and ignore them here.
+# At that point we will rely the frozen manifest file to identify
+# changed generated files. We'll drop the entry for it then.
+# See: Tools/scripts/freeze_modules.py.
+#Python/frozen_modules/*.h
+
 # Two-trick pony for OSX and other case insensitive file systems:
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python

Doc/using/cmdline.rst

@@ -480,6 +480,13 @@ Miscellaneous options
  objects and pyc files are desired as well as supressing the extra visual
  location indicators when the interpreter displays tracebacks. See also
  :envvar:`PYTHONNODEBUGRANGES`.
+ * ``-X frozen_modules=[on|off]`` determines whether or not frozen modules
+ are ignored by the import machinery. A value of "on" means they get
+ imported and "off" means they are ignored. The default is "on"
+ if this is an installed Python (the normal case). If it's under
+ development (running from the build dir) then the default is "off".
+ Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
+ frozen modules are always used, even if this flag is set to "off".
 
  It also allows passing arbitrary values and retrieving them through the
  :data:`sys._xoptions` dictionary.
@@ -518,6 +525,9 @@ Miscellaneous options
  .. versionadded:: 3.11
  The ``-X no_debug_ranges`` option.
 
+ .. versionadded:: 3.11
+ The ``-X frozen_modules`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -2,6 +2,8 @@
 #define Py_PYCORECONFIG_H
 #ifndef Py_LIMITED_API
 
+#include <stdbool.h>
+
 /* --- PyStatus ----------------------------------------------- */
 
 typedef struct {
@@ -172,6 +174,7 @@ typedef struct PyConfig {
  int legacy_windows_stdio;
 #endif
  wchar_t *check_hash_pycs_mode;
+ bool use_frozen_modules;
 
  /* --- Path configuration inputs ------------ */
  int pathconfig_warnings;

Include/internal/pycore_initconfig.h

@@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy(
 extern PyStatus _PyConfig_InitPathConfig(
  PyConfig *config,
  int compute_path_config);
+extern PyStatus _PyConfig_InitImportConfig(PyConfig *config);
 extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
  struct pyruntimestate *runtime);

Include/internal/pycore_pylifecycle.h

@@ -12,6 +12,7 @@ extern "C" {
 #include <signal.h>
 #endif
 
+#include <stdbool.h>
 #include "pycore_runtime.h" // _PyRuntimeState
 
 #ifndef NSIG
@@ -122,7 +123,6 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig(
  const PyConfig *config,
  const struct _PyArgv *args);
 
-
 PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
 
 PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable);

Include/internal/pycore_pystate.h

@@ -30,6 +30,17 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
 }
 
 
+static inline const PyConfig *
+_Py_GetMainConfig(void)
+{
+ PyInterpreterState *interp = _PyRuntime.interpreters.main;
+ if (interp == NULL) {
+ return NULL;
+ }
+ return _PyInterpreterState_GetConfig(interp);
+}
+
+
 /* Only handle signals on the main thread of the main interpreter. */
 static inline int
 _Py_ThreadCanHandleSignals(PyInterpreterState *interp)

Lib/ctypes/test/test_values.py

@@ -2,9 +2,12 @@
 A testcase which accesses *values* in a dll.
 """
 
+import imp
+import importlib.util
 import unittest
 import sys
 from ctypes import *
+from test.support import import_helper, captured_stdout
 
 import _ctypes_test
 
@@ -55,41 +58,34 @@ class struct_frozen(Structure):
 
  ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
  # ft is a pointer to the struct_frozen entries:
- items = []
- # _frozen_importlib changes size whenever importlib._bootstrap
- # changes, so it gets a special case. We should make sure it's
- # found, but don't worry about its size too much. The same
- # applies to _frozen_importlib_external.
- bootstrap_seen = []
- bootstrap_expected = [
- b'_frozen_importlib',
- b'_frozen_importlib_external',
- b'zipimport',
- ]
+ modules = []
  for entry in ft:
  # This is dangerous. We *can* iterate over a pointer, but
  # the loop will not terminate (maybe with an access
  # violation;-) because the pointer instance has no size.
  if entry.name is None:
  break
-
- if entry.name in bootstrap_expected:
- bootstrap_seen.append(entry.name)
- self.assertTrue(entry.size,
- "{!r} was reported as having no size".format(entry.name))
- continue
- items.append((entry.name.decode("ascii"), entry.size))
-
- expected = [("__hello__", 164),
- ("__phello__", -164),
- ("__phello__.spam", 164),
- ]
- self.assertEqual(items, expected, "PyImport_FrozenModules example "
+ modname = entry.name.decode("ascii")
+ modules.append(modname)
+ with self.subTest(modname):
+ # Do a sanity check on entry.size and entry.code.
+ self.assertGreater(abs(entry.size), 10)
+ self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
+ # Check the module's package-ness.
+ with import_helper.frozen_modules(), captured_stdout():
+ spec = importlib.util.find_spec(modname)
+ if entry.size < 0:
+ # It's a package.
+ self.assertIsNotNone(spec.submodule_search_locations)
+ else:
+ self.assertIsNone(spec.submodule_search_locations)
+
+ with import_helper.frozen_modules():
+ expected = imp._frozen_module_names()
+ self.maxDiff = None
+ self.assertEqual(modules, expected, "PyImport_FrozenModules example "
  "in Doc/library/ctypes.rst may be out of date")
 
- self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
- "frozen bootstrap modules did not match PyImport_FrozenModules")
-
  from ctypes import _pointer_type_cache
  del _pointer_type_cache[struct_frozen]
 

Lib/idlelib/idle_test/test_query.py

@@ -136,8 +136,8 @@ def test_good_module_name(self):
  dialog = self.Dummy_ModuleName('idlelib')
  self.assertTrue(dialog.entry_ok().endswith('__init__.py'))
  self.assertEqual(dialog.entry_error['text'], '')
- dialog = self.Dummy_ModuleName('os.path')
- self.assertTrue(dialog.entry_ok().endswith('path.py'))
+ dialog = self.Dummy_ModuleName('idlelib.idle')
+ self.assertTrue(dialog.entry_ok().endswith('idle.py'))
  self.assertEqual(dialog.entry_error['text'], '')
 
 

Lib/imp.py

@@ -9,7 +9,7 @@
 from _imp import (lock_held, acquire_lock, release_lock,
  get_frozen_object, is_frozen_package,
  init_frozen, is_builtin, is_frozen,
- _fix_co_filename)
+ _fix_co_filename, _frozen_module_names)
 try:
  from _imp import create_dynamic
 except ImportError:

Lib/test/support/import_helper.py

@@ -109,7 +109,20 @@ def _save_and_block_module(name, orig_modules):
  return saved
 
 
-def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+@contextlib.contextmanager
+def frozen_modules(enabled=True):
+ # FYI: the env var will never show up in os.environ.
+ os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0')
+ try:
+ yield
+ finally:
+ os.unsetenv('_PYTHONTESTFROZENMODULES')
+
+
+def import_fresh_module(name, fresh=(), blocked=(), *,
+ deprecated=False,
+ usefrozen=False,
+ ):
  """Import and return a module, deliberately bypassing sys.modules.
 
  This function imports and returns a fresh copy of the named Python module
@@ -148,7 +161,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
  for blocked_name in blocked:
  if not _save_and_block_module(blocked_name, orig_modules):
  names_to_remove.append(blocked_name)
- fresh_module = importlib.import_module(name)
+ with frozen_modules(usefrozen):
+ fresh_module = importlib.import_module(name)
  except ImportError:
  fresh_module = None
  finally:
@@ -171,7 +185,7 @@ class CleanImport(object):
  importlib.import_module("foo") # new reference
  """
 
- def __init__(self, *module_names):
+ def __init__(self, *module_names, usefrozen=False):
  self.original_modules = sys.modules.copy()
  for module_name in module_names:
  if module_name in sys.modules:
@@ -183,12 +197,15 @@ def __init__(self, *module_names):
  if module.__name__ != module_name:
  del sys.modules[module.__name__]
  del sys.modules[module_name]
+ self._frozen_modules = frozen_modules(usefrozen)
 
  def __enter__(self):
+ self._frozen_modules.__enter__()
  return self
 
  def __exit__(self, *ignore_exc):
  sys.modules.update(self.original_modules)
+ self._frozen_modules.__exit__(*ignore_exc)
 
 
 class DirsOnSysPath(object):

Lib/test/support/os_helper.py

@@ -599,6 +599,10 @@ def set(self, envvar, value):
  def unset(self, envvar):
  del self[envvar]
 
+ def copy(self):
+ # We do what os.environ.copy() does.
+ return dict(self)
+
  def __enter__(self):
  return self
 

Lib/test/test_cmd_line_script.py

@@ -474,7 +474,6 @@ def test_dash_m_errors(self):
  br'ModuleNotFoundError'),
  ('builtins.x.y', br'Error while finding module specification.*'
  br'ModuleNotFoundError.*No module named.*not a package'),
- ('os.path', br'loader.*cannot handle'),
  ('importlib', br'No module named.*'
  br'is a package and cannot be directly executed'),
  ('importlib.nonexistent', br'No module named'),

Lib/test/test_embed.py

@@ -12,6 +12,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -426,11 +427,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  'pathconfig_warnings': 1,
  '_init_main': 1,
  '_isolated_interpreter': 0,
+ 'use_frozen_modules': False,
  }
  if MS_WINDOWS:
  CONFIG_COMPAT.update({
  'legacy_windows_stdio': 0,
  })
+ else:
+ config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
+ if '--with-address-sanitizer' in config_args:
+ CONFIG_COMPAT['use_frozen_modules'] = True
 
  CONFIG_PYTHON = dict(CONFIG_COMPAT,
  _config_init=API_PYTHON,
@@ -1244,7 +1250,9 @@ def test_init_setpythonhome(self):
  'pythonpath_env': paths_str,
  }
  self.default_program_name(config)
- env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+ env = {'TESTHOME': home,
+ 'PYTHONPATH': paths_str,
+ '_PYTHONTESTFROZENMODULES': '1'}
  self.check_all_configs("test_init_setpythonhome", config,
  api=API_COMPAT, env=env)
 

Lib/test/test_frozen.py

@@ -12,16 +12,17 @@
 
 import sys
 import unittest
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
 
 
 class TestFrozen(unittest.TestCase):
  def test_frozen(self):
  name = '__hello__'
  if name in sys.modules:
  del sys.modules[name]
- with captured_stdout() as out:
- import __hello__
+ with import_helper.frozen_modules():
+ with captured_stdout() as out:
+ import __hello__
  self.assertEqual(out.getvalue(), 'Hello world!\n')
 
 

Lib/test/test_imp.py

@@ -16,6 +16,9 @@
 import _imp
 
 
+OS_PATH_NAME = os.path.__name__
+
+
 def requires_load_dynamic(meth):
  """Decorator to skip a test if not running under CPython or lacking
  imp.load_dynamic()."""
@@ -213,15 +216,17 @@ def test_load_from_source(self):
  # state after reversion. Reinitialising the module contents
  # and just reverting os.environ to its previous state is an OK
  # workaround
- orig_path = os.path
- orig_getenv = os.getenv
- with os_helper.EnvironmentVarGuard():
- x = imp.find_module("os")
- self.addCleanup(x[0].close)
- new_os = imp.load_module("os", *x)
- self.assertIs(os, new_os)
- self.assertIs(orig_path, new_os.path)
- self.assertIsNot(orig_getenv, new_os.getenv)
+ with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
+ import os
+ orig_path = os.path
+ orig_getenv = os.getenv
+ with os_helper.EnvironmentVarGuard():
+ x = imp.find_module("os")
+ self.addCleanup(x[0].close)
+ new_os = imp.load_module("os", *x)
+ self.assertIs(os, new_os)
+ self.assertIs(orig_path, new_os.path)
+ self.assertIsNot(orig_getenv, new_os.getenv)
 
  @requires_load_dynamic
  def test_issue15828_load_extensions(self):

Lib/test/test_import/__init__.py

@@ -21,7 +21,7 @@
 from test.support import os_helper
 from test.support import (is_jython, swap_attr, swap_item, cpython_only)
 from test.support.import_helper import (
- forget, make_legacy_pyc, unlink, unload, DirsOnSysPath)
+ forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
 from test.support.os_helper import (
  TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir)
 from test.support import script_helper
@@ -86,8 +86,10 @@ def test_from_import_missing_attr_raises_ImportError(self):
  from importlib import something_that_should_not_exist_anywhere
 
  def test_from_import_missing_attr_has_name_and_path(self):
- with self.assertRaises(ImportError) as cm:
- from os import i_dont_exist
+ with CleanImport('os'):
+ import os
+ with self.assertRaises(ImportError) as cm:
+ from os import i_dont_exist
  self.assertEqual(cm.exception.name, 'os')
  self.assertEqual(cm.exception.path, os.__file__)
  self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)")