gh-69605: Hardcode some stdlib submodules in PyREPL module completion (os.path, collections.abc...)

Lib/_pyrepl/_module_completer.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import importlib
+import os
 import pkgutil
 import sys
 import token
 import tokenize
+from importlib.machinery import FileFinder
 from io import StringIO
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -16,6 +19,15 @@
  from typing import Any, Iterable, Iterator, Mapping
 
 
+HARDCODED_SUBMODULES = {
+ # Standard library submodules that are not detected by pkgutil.iter_modules
+ # but can be imported, so should be proposed in completion
+ "collections": ["abc"],
+ "os": ["path"],
+ "xml.parsers.expat": ["errors", "model"],
+}
+
+
 def make_default_module_completer() -> ModuleCompleter:
  # Inside pyrepl, __package__ is set to None by default
  return ModuleCompleter(namespace={'__package__': None})
@@ -41,6 +53,7 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None:
  self.namespace = namespace or {}
  self._global_cache: list[pkgutil.ModuleInfo] = []
  self._curr_sys_path: list[str] = sys.path[:]
+ self._stdlib_path = os.path.dirname(importlib.__path__[0])
 
  def get_completions(self, line: str) -> list[str] | None:
  """Return the next possible import completions for 'line'."""
@@ -95,12 +108,26 @@ def _find_modules(self, path: str, prefix: str) -> list[str]:
  return []
 
  modules: Iterable[pkgutil.ModuleInfo] = self.global_cache
+ is_stdlib_import: bool | None = None
  for segment in path.split('.'):
  modules = [mod_info for mod_info in modules
  if mod_info.ispkg and mod_info.name == segment]
+ if is_stdlib_import is None:
+ # Top-level import decide if we import from stdlib or not
+ is_stdlib_import = all(
+ self._is_stdlib_module(mod_info) for mod_info in modules
+ )
  modules = self.iter_submodules(modules)
- return [module.name for module in modules
- if self.is_suggestion_match(module.name, prefix)]
+
+ module_names = [module.name for module in modules]
+ if is_stdlib_import:
+ module_names.extend(HARDCODED_SUBMODULES.get(path, ()))
+ return [module_name for module_name in module_names
+ if self.is_suggestion_match(module_name, prefix)]
+
+ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool:
+ return (isinstance(module_info.module_finder, FileFinder)
+ and module_info.module_finder.path == self._stdlib_path)
 
  def is_suggestion_match(self, module_name: str, prefix: str) -> bool:
  if prefix:

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1,3 +1,4 @@
+import importlib
 import io
 import itertools
 import os
@@ -26,9 +27,16 @@
  code_to_events,
 )
 from _pyrepl.console import Event
-from _pyrepl._module_completer import ImportParser, ModuleCompleter
-from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig,
- _ReadlineWrapper)
+from _pyrepl._module_completer import (
+ ImportParser,
+ ModuleCompleter,
+ HARDCODED_SUBMODULES,
+)
+from _pyrepl.readline import (
+ ReadlineAlikeReader,
+ ReadlineConfig,
+ _ReadlineWrapper,
+)
 from _pyrepl.readline import multiline_input as readline_multiline_input
 
 try:
@@ -930,7 +938,6 @@ def test_func(self):
 
 class TestPyReplModuleCompleter(TestCase):
  def setUp(self):
- import importlib
  # Make iter_modules() search only the standard library.
  # This makes the test more reliable in case there are
  # other user packages/scripts on PYTHONPATH which can
@@ -1013,14 +1020,6 @@ def test_sub_module_private_completions(self):
  self.assertEqual(output, expected)
 
  def test_builtin_completion_top_level(self):
- import importlib
- # Make iter_modules() search only the standard library.
- # This makes the test more reliable in case there are
- # other user packages/scripts on PYTHONPATH which can
- # intefere with the completions.
- lib_path = os.path.dirname(importlib.__path__[0])
- sys.path = [lib_path]
-
  cases = (
  ("import bui\t\n", "import builtins"),
  ("from bui\t\n", "from builtins"),
@@ -1076,6 +1075,32 @@ def test_no_fallback_on_regular_completion(self):
  output = reader.readline()
  self.assertEqual(output, expected)
 
+ def test_hardcoded_stdlib_submodules(self):
+ cases = (
+ ("import collections.\t\n", "import collections.abc"),
+ ("from os import \t\n", "from os import path"),
+ ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"),
+ ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"),
+ )
+ for code, expected in cases:
+ with self.subTest(code=code):
+ events = code_to_events(code)
+ reader = self.prepare_reader(events, namespace={})
+ output = reader.readline()
+ self.assertEqual(output, expected)
+
+ def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self):
+ with tempfile.TemporaryDirectory() as _dir:
+ dir = pathlib.Path(_dir)
+ (dir / "collections").mkdir()
+ (dir / "collections" / "__init__.py").touch()
+ (dir / "collections" / "foo.py").touch()
+ with patch.object(sys, "path", [dir, *sys.path]):
+ events = code_to_events("import collections.\t\n")
+ reader = self.prepare_reader(events, namespace={})
+ output = reader.readline()
+ self.assertEqual(output, "import collections.foo")
+
  def test_get_path_and_prefix(self):
  cases = (
  ('', ('', '')),
@@ -1204,6 +1229,19 @@ def test_parse_error(self):
  with self.subTest(code=code):
  self.assertEqual(actual, None)
 
+
+class TestHardcodedSubmodules(TestCase):
+ def test_hardcoded_stdlib_submodules_are_importable(self):
+ for parent_path, submodules in HARDCODED_SUBMODULES.items():
+ for module_name in submodules:
+ path = f"{parent_path}.{module_name}"
+ with self.subTest(path=path):
+ # We can't use importlib.util.find_spec here,
+ # since some hardcoded submodules parents are
+ # not proper packages
+ importlib.import_module(path)
+
+
 class TestPasteEvent(TestCase):
  def prepare_reader(self, events):
  console = FakeConsole(events)

Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst

@@ -0,0 +1 @@
+Fix some standard library submodules missing from the :term:`REPL` auto-completion of imports.