Fix mypy_test if mypy returns negative exit code

tests/mypy_test.py

@@ -13,6 +13,7 @@
 import time
 from collections import defaultdict
 from dataclasses import dataclass
+from enum import Enum
 from itertools import product
 from pathlib import Path
 from threading import Lock
@@ -51,7 +52,6 @@
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
 DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
 
-ReturnCode: TypeAlias = int
 VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
 VersionTuple: TypeAlias = Tuple[int, int]
 Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
@@ -222,6 +222,12 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) ->
  configurations.append(MypyDistConf(module_name, values.copy()))
 
 
+class MypyResult(Enum):
+ SUCCESS = 0
+ FAILURE = 1
+ CRASH = 2
+
+
 def run_mypy(
  args: TestConfig,
  configurations: list[MypyDistConf],
@@ -231,7 +237,7 @@ def run_mypy(
  non_types_dependencies: bool,
  venv_info: VenvInfo,
  mypypath: str | None = None,
-) -> ReturnCode:
+) -> MypyResult:
  env_vars = dict(os.environ)
  if mypypath is not None:
  env_vars["MYPYPATH"] = mypypath
@@ -276,7 +282,7 @@ def run_mypy(
  print(colored(f"running {' '.join(mypy_command)}", "blue"))
  result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
  if result.returncode:
- print_error("failure\n")
+ print_error(f"failure (exit code {result.returncode})\n")
  if result.stdout:
  print_error(result.stdout)
  if result.stderr:
@@ -287,7 +293,12 @@ def run_mypy(
  print()
  else:
  print_success_msg()
- return result.returncode
+ if result.returncode == 0:
+ return MypyResult.SUCCESS
+ elif result.returncode == 1:
+ return MypyResult.FAILURE
+ else:
+ return MypyResult.CRASH
 
 
 def add_third_party_files(
@@ -305,15 +316,14 @@ def add_third_party_files(
  add_configuration(configurations, distribution)
 
 
-class TestResults(NamedTuple):
-exit_code: int
+class TestResult(NamedTuple):
+mypy_result: MypyResult
  files_checked: int
- packages_skipped: int = 0
 
 
 def test_third_party_distribution(
  distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool
-) -> TestResults:
+) -> TestResult:
  """Test the stubs of a third-party distribution.
 
  Return a tuple, where the first element indicates mypy's return code
@@ -326,7 +336,7 @@ def test_third_party_distribution(
  add_third_party_files(distribution, files, args, configurations, seen_dists)
 
  if not files and args.filter:
- return TestResults(0, 0)
+ return TestResult(MypyResult.SUCCESS, 0)
 
  print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True)
 
@@ -337,7 +347,7 @@ def test_third_party_distribution(
  mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists)
  if args.verbose:
  print(colored(f"\nMYPYPATH={mypypath}", "blue"))
-code = run_mypy(
+result = run_mypy(
  args,
  configurations,
  files,
@@ -346,10 +356,10 @@ def test_third_party_distribution(
  testing_stdlib=False,
  non_types_dependencies=non_types_dependencies,
  )
- return TestResults(code, len(files))
+ return TestResult(result, len(files))
 
 
-def test_stdlib(code: int, args: TestConfig) -> TestResults:
+def test_stdlib(args: TestConfig) -> TestResult:
  files: list[Path] = []
  stdlib = Path("stdlib")
  supported_versions = parse_versions(stdlib / "VERSIONS")
@@ -361,14 +371,39 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults:
  if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version:
  add_files(files, (stdlib / name), args)
 
- if files:
- print(f"Testing stdlib ({len(files)} files)...", end="", flush=True)
- # We don't actually need pip for the stdlib testing
- venv_info = VenvInfo(pip_exe="", python_exe=sys.executable)
- this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False)
- code = max(code, this_code)
+ if not files:
+ return TestResult(MypyResult.SUCCESS, 0)
+
+ print(f"Testing stdlib ({len(files)} files)...", end="", flush=True)
+ # We don't actually need pip for the stdlib testing
+ venv_info = VenvInfo(pip_exe="", python_exe=sys.executable)
+ result = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False)
+ return TestResult(result, len(files))
 
- return TestResults(code, len(files))
+
+@dataclass
+class TestSummary:
+ mypy_result: MypyResult = MypyResult.SUCCESS
+ files_checked: int = 0
+ packages_skipped: int = 0
+ packages_with_errors: int = 0
+
+ def register_result(self, mypy_result: MypyResult, files_checked: int) -> None:
+ if mypy_result.value > self.mypy_result.value:
+ self.mypy_result = mypy_result
+ if mypy_result != MypyResult.SUCCESS:
+ self.packages_with_errors += 1
+ self.files_checked += files_checked
+
+ def skip_package(self) -> None:
+ self.packages_skipped += 1
+
+ def merge(self, other: TestSummary) -> None:
+ if other.mypy_result.value > self.mypy_result.value:
+ self.mypy_result = other.mypy_result
+ self.files_checked += other.files_checked
+ self.packages_skipped += other.packages_skipped
+ self.packages_with_errors += other.packages_with_errors
 
 
 _PRINT_LOCK = Lock()
@@ -473,10 +508,9 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar
  _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use))
 
 
-def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults:
+def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary:
  print("Testing third-party packages...")
- files_checked = 0
- packages_skipped = 0
+ summary = TestSummary()
  gitignore_spec = get_gitignore_spec()
  distributions_to_check: dict[str, PackageDependencies] = {}
 
@@ -493,12 +527,12 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe
  f"test is being run using Python {PYTHON_VERSION})"
  )
  print(colored(msg, "yellow"))
-packages_skipped += 1
+summary.skip_package()
  continue
  if not metadata.requires_python.contains(args.version):
  msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})"
  print(colored(msg, "yellow"))
-packages_skipped += 1
+summary.skip_package()
  continue
 
  if (
@@ -527,32 +561,30 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe
  for distribution in distributions_to_check:
  venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution]
  non_types_dependencies = venv_info.python_exe != sys.executable
-this_code, checked, _ = test_third_party_distribution(
+mypy_result, files_checked = test_third_party_distribution(
  distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies
  )
- code = max(code, this_code)
- files_checked += checked
+ summary.register_result(mypy_result, files_checked)
 
- return TestResults(code, files_checked, packages_skipped)
+ return summary
 
 
-def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults:
+def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary:
  print(f"*** Testing Python {args.version} on {args.platform}")
- files_checked_this_version = 0
- packages_skipped_this_version = 0
  stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs")
+ summary = TestSummary()
+
  if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter):
-code, stdlib_files_checked, _ = test_stdlib(code, args)
-files_checked_this_version += stdlib_files_checked
+mypy_result, files_checked = test_stdlib(args)
+summary.register_result(mypy_result, files_checked)
  print()
 
  if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter):
- code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir)
- files_checked_this_version += third_party_files_checked
- packages_skipped_this_version = third_party_packages_skipped
+ tp_results = test_third_party_stubs(args, tempdir)
+ summary.merge(tp_results)
  print()
 
- return TestResults(code, files_checked_this_version, packages_skipped_this_version)
+ return summary
 
 
 def main() -> None:
@@ -561,26 +593,31 @@ def main() -> None:
  platforms = args.platform or [sys.platform]
  filter = args.filter or DIRECTORIES_TO_TEST
  exclude = args.exclude or []
- code = 0
- total_files_checked = 0
- total_packages_skipped = 0
+ summary = TestSummary()
  with tempfile.TemporaryDirectory() as td:
  td_path = Path(td)
  for version, platform in product(versions, platforms):
  config = TestConfig(args.verbose, filter, exclude, version, platform)
- code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path)
- total_files_checked += files_checked_this_version
- total_packages_skipped += packages_skipped_this_version
- if code:
- plural = "" if total_files_checked == 1 else "s"
- print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---")
- sys.exit(code)
- if total_packages_skipped:
- plural = "" if total_packages_skipped == 1 else "s"
- print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow"))
- if total_files_checked:
- plural = "" if total_files_checked == 1 else "s"
- print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green"))
+ version_summary = test_typeshed(args=config, tempdir=td_path)
+ summary.merge(version_summary)
+
+ if summary.mypy_result == MypyResult.FAILURE:
+ plural1 = "" if summary.packages_with_errors == 1 else "s"
+ plural2 = "" if summary.files_checked == 1 else "s"
+ print_error(
+ f"--- {summary.packages_with_errors} package{plural1} with errors, {summary.files_checked} file{plural2} checked ---"
+ )
+ sys.exit(1)
+ if summary.mypy_result == MypyResult.CRASH:
+ plural = "" if summary.files_checked == 1 else "s"
+ print_error(f"--- mypy crashed, {summary.files_checked} file{plural} checked ---")
+ sys.exit(2)
+ if summary.packages_skipped:
+ plural = "" if summary.packages_skipped == 1 else "s"
+ print(colored(f"--- {summary.packages_skipped} package{plural} skipped ---", "yellow"))
+ if summary.files_checked:
+ plural = "" if summary.files_checked == 1 else "s"
+ print(colored(f"--- success, {summary.files_checked} file{plural} checked ---", "green"))
  else:
  print_error("--- nothing to do; exit 1 ---")
  sys.exit(1)