Further improve non-types dependency handling in pytype_test.py

.github/workflows/typecheck_typeshed_code.yml

@@ -51,7 +51,7 @@ jobs:
  - uses: actions/checkout@v3
  - uses: actions/setup-python@v4
  with:
- python-version: "3.10"
+ python-version: "3.9"
  cache: pip
  cache-dependency-path: requirements-tests.txt
  - run: pip install -r requirements-tests.txt
@@ -66,5 +66,5 @@ jobs:
  with:
  version: ${{ steps.pyright_version.outputs.value }}
  python-platform: ${{ matrix.python-platform }}
- python-version: "3.10"
+ python-version: "3.9"
  project: ./pyrightconfig.scripts_and_tests.json

tests/README.md

@@ -19,7 +19,7 @@ objects at runtime.
 in the `tests` and `scripts` directories.
 
 To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
-in the `CONTRIBUTING.md` document. In particular, you have to run with Python 3.10+.
+in the `CONTRIBUTING.md` document. In particular, you have to run with Python 3.9+.
 
 In order for `pytype_test` and `pyright_test` to work correctly, some third-party stubs
 may require extra dependencies external to typeshed to be installed in your virtual environment
@@ -72,7 +72,7 @@ for this script.
 
 Note: this test cannot be run on Windows
 systems unless you are using Windows Subsystem for Linux.
-It can currently only be run on Python 3.10 as pytype does not yet support
+It also requires a Python version < 3.11 as pytype does not yet support
 Python 3.11 and above.
 
 Run using:

tests/pytype_test.py

@@ -16,10 +16,10 @@
 
 import argparse
 import importlib.metadata
+import inspect
 import os
 import sys
 import traceback
-from collections import defaultdict
 from collections.abc import Iterable, Sequence
 
 from packaging.requirements import Requirement
@@ -29,8 +29,8 @@
 if sys.platform == "win32":
  print("pytype does not support Windows.", file=sys.stderr)
  sys.exit(1)
-if sys.version_info[:2] != (3, 10):
- print("pytype_test.py can currently only be run on Python 3.10.", file=sys.stderr)
+if sys.version_info >= (3, 11):
+ print("pytype does not support Python 3.11+ yet.", file=sys.stderr)
  sys.exit(1)
 
 # pytype is not py.typed https://github.com/google/pytype/issues/1325
@@ -149,6 +149,17 @@ def find_stubs_in_paths(paths: Sequence[str]) -> list[str]:
  return filenames
 
 
+def _get_pkgs_associated_with_requirement(req_name: str) -> list[str]:
+ dist = importlib.metadata.distribution(req_name)
+ toplevel_txt_contents = dist.read_text("top_level.txt")
+ if toplevel_txt_contents is not None:
+ return toplevel_txt_contents.split()
+ if dist.files is None:
+ raise RuntimeError("Can't read find the packages associated with requirement {req_name!r}")
+ maybe_modules = [f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) for f in dist.files]
+ return [name for name in maybe_modules if name is not None and "." not in name]
+
+
 def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]:
  """Get names of modules that should be treated as missing.
 
@@ -168,16 +179,12 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]:
  continue
  stub_distributions.add(parts[idx + 1])
 
- dist_to_pkg_map = defaultdict(list)
- for dist, pkg_list in importlib.metadata.packages_distributions().items():
- for pkg in pkg_list:
- dist_to_pkg_map[pkg].append(dist)
-
  missing_modules = set()
  for distribution in stub_distributions:
  for external_req in read_dependencies(distribution).external_pkgs:
- pkg = Requirement(external_req).name
- missing_modules.update(dist_to_pkg_map[pkg])
+ req_name = Requirement(external_req).name
+ associated_packages = _get_pkgs_associated_with_requirement(req_name)
+ missing_modules.update(associated_packages)
 
  test_dir = os.path.dirname(__file__)
  exclude_list = os.path.join(test_dir, "pytype_exclude_list.txt")

tests/typecheck_typeshed.py

@@ -13,7 +13,7 @@
 ReturnCode: TypeAlias = int
 
 SUPPORTED_PLATFORMS = ("linux", "darwin", "win32")
-SUPPORTED_VERSIONS = ("3.12", "3.11", "3.10")
+SUPPORTED_VERSIONS = ("3.12", "3.11", "3.10", "3.9")
 LOWEST_SUPPORTED_VERSION = min(SUPPORTED_VERSIONS, key=lambda x: int(x.split(".")[1]))
 DIRECTORIES_TO_TEST = ("scripts", "tests")
 EMPTY: list[str] = []
@@ -63,6 +63,8 @@ def run_mypy_as_subprocess(directory: str, platform: str, version: str) -> Retur
  "possibly-undefined",
  "--enable-error-code",
  "redundant-expr",
+ "--custom-typeshed-dir",
+ ".",
  ]
  if directory == "tests" and platform == "win32":
  command.extend(["--exclude", "tests/pytype_test.py"])