diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f31363..6e85c34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Analysis env provisioning is parso-version-aware** (#107): on hosts whose default + `python3` is newer than the newest grammar the installed parso ships (e.g. Python 3.14 + with parso ≤ 0.8.4), every file failed jedi parsing and the run completed "successfully" + with an empty symbol table. Provisioning now derives parso's supported ceiling at runtime + from its shipped grammar files and prefers the newest supported interpreter on the host + (versioned PATH names, then pyenv installs), falling back loudly only when none exists; + an explicit `SYSTEM_PYTHON` is still honored, with a warning when unsupported. A run in + which every discovered file fails now logs a prominent error instead of staying silent, + and `parso>=0.8.5` (the first release with the 3.14 grammar) is a direct dependency. + ## [1.0.2] - 2026-07-16 ### Fixed diff --git a/codeanalyzer/core.py b/codeanalyzer/core.py index a533d0f..05b464d 100644 --- a/codeanalyzer/core.py +++ b/codeanalyzer/core.py @@ -163,8 +163,159 @@ def _cmd_exec_helper( stderr=None, ) + @classmethod + def _get_base_interpreter(cls) -> Path: + """The interpreter used to provision the analysis virtualenv. + + jedi parses the *analysis environment's* Python version with parso, + which ships one hardcoded grammar file per minor version — an + environment newer than the newest shipped grammar makes every file + fail with "Python version X.Y is currently not supported" while the + run still exits 0 (#107). So the default choice is gated on the + installed parso's ceiling: a too-new default is swapped for the + newest supported interpreter found on the host, falling back to the + default (loudly) only when none exists. An explicit ``SYSTEM_PYTHON`` + always wins, with a warning when parso cannot parse its version. + """ + # An explicit SYSTEM_PYTHON override wins (consulted only when running + # inside a virtualenv, matching the historical behavior). + if sys.prefix != sys.base_prefix: + system_python = os.getenv("SYSTEM_PYTHON") + if system_python: + system_python_path = Path(system_python) + if system_python_path.exists() and system_python_path.is_file(): + ceiling = cls._parso_supported_ceiling() + version = cls._interpreter_version(system_python_path) + if ceiling is not None and version is not None and version > ceiling: + logger.warning( + f"SYSTEM_PYTHON={system_python} is Python " + f"{version[0]}.{version[1]}, newer than the newest grammar " + f"the installed parso ships ({ceiling[0]}.{ceiling[1]}). " + "jedi will likely reject every file in the analysis " + "environment (#107); honoring the explicit override anyway." + ) + return system_python_path + + candidate = cls._default_base_interpreter() + ceiling = cls._parso_supported_ceiling() + if ceiling is None: + return candidate + version = cls._interpreter_version(candidate) + if version is None or version <= ceiling: + return candidate + logger.warning( + f"Default interpreter {candidate} is Python {version[0]}.{version[1]}, " + f"newer than the newest grammar the installed parso ships " + f"({ceiling[0]}.{ceiling[1]}) — looking for a supported interpreter " + "for the analysis environment (#107)." + ) + supported = cls._find_supported_interpreter(ceiling) + if supported is not None: + logger.info(f"Provisioning the analysis environment with {supported}.") + return supported + logger.warning( + f"No interpreter <= {ceiling[0]}.{ceiling[1]} found on this host; " + f"falling back to {candidate}. jedi/parso will likely reject every " + "file — install a supported Python or upgrade parso." + ) + return candidate + + @staticmethod + def _versions_from_grammar_stems(stems: List[str]) -> List[tuple]: + """``grammar313`` → ``(3, 13)``, sorted ascending; malformed stems dropped.""" + versions = [] + for stem in stems: + digits = stem[len("grammar"):] + if len(digits) >= 2 and digits.isdigit(): + versions.append((int(digits[0]), int(digits[1:]))) + return sorted(versions) + + @classmethod + def _parso_supported_ceiling(cls) -> Optional[tuple]: + """Newest ``(major, minor)`` the installed parso ships a grammar for, + derived from its ``python/grammar*.txt`` files so the ceiling moves + automatically when parso adds a version. ``None`` if undeterminable.""" + try: + import parso + + stems = [ + p.stem + for p in (Path(parso.__file__).parent / "python").glob("grammar*.txt") + ] + versions = cls._versions_from_grammar_stems(stems) + return versions[-1] if versions else None + except Exception: + return None + + @staticmethod + def _interpreter_version(interpreter: Path) -> Optional[tuple]: + """``(major, minor)`` of an interpreter, or ``None`` if it can't run.""" + try: + result = subprocess.run( + [ + str(interpreter), + "-c", + "import sys; print('%d.%d' % sys.version_info[:2])", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + major, minor = result.stdout.strip().split(".") + return (int(major), int(minor)) + except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, ValueError): + pass + return None + + @staticmethod + def _pick_supported_interpreter( + candidates: List[tuple], ceiling: tuple + ) -> Optional[Path]: + """Newest candidate whose version is within the ceiling. + + ``candidates`` is ``[(path, (major, minor) | None), ...]``.""" + supported = [ + (version, path) + for path, version in candidates + if version is not None and version <= ceiling + ] + return max(supported)[1] if supported else None + + @classmethod + def _find_supported_interpreter(cls, ceiling: tuple) -> Optional[Path]: + """Search the host for the newest interpreter within the parso ceiling: + versioned names on PATH (``python3.13``, ``python3.12``, ...) first, + then pyenv installs.""" + paths: List[Path] = [] + for minor in range(ceiling[1], 7, -1): + which = shutil.which(f"python{ceiling[0]}.{minor}") + # Skip the current virtualenv's own interpreter (same rule as + # _default_base_interpreter): the analysis env must come from a + # base installation. + if which and not which.startswith(sys.prefix): + paths.append(Path(which)) + for pyenv_root in (os.getenv("PYENV_ROOT"), str(Path.home() / ".pyenv")): + if not pyenv_root: + continue + versions_dir = Path(pyenv_root) / "versions" + if versions_dir.is_dir(): + for install in sorted(versions_dir.iterdir(), reverse=True): + exe = install / "bin" / "python3" + if exe.exists(): + paths.append(exe) + seen = set() + candidates = [] + for path in paths: + key = str(path) + if key in seen: + continue + seen.add(key) + candidates.append((path, cls._interpreter_version(path))) + return cls._pick_supported_interpreter(candidates, ceiling) + @staticmethod - def _get_base_interpreter() -> Path: + def _default_base_interpreter() -> Path: """Get the base Python interpreter path. This method finds a suitable base Python interpreter that can be used @@ -183,13 +334,6 @@ def _get_base_interpreter() -> Path: # We're inside a virtual environment; need to find the base interpreter - # First, check if user explicitly set SYSTEM_PYTHON - system_python = os.getenv("SYSTEM_PYTHON") - if system_python: - system_python_path = Path(system_python) - if system_python_path.exists() and system_python_path.is_file(): - return system_python_path - # Try to get the base interpreter from sys.base_executable (Python 3.3+) if hasattr(sys, "base_executable") and sys.base_executable: base_exec = Path(sys.base_executable) @@ -778,6 +922,15 @@ def _build_symbol_table(self, cached_symbol_table: Optional[Dict[str, PyModule]] if files_from_cache > 0: logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files") + if py_files and not symbol_table: + logger.error( + "Every one of the %d discovered Python files failed to process — " + "the symbol table is empty. This usually means the analysis " + "environment's interpreter is newer than the installed jedi/parso " + "stack supports (#107); check the per-file errors above.", + len(py_files), + ) + logger.info( "✅ Symbol table: %d modules in %.1fs", len(symbol_table), time.perf_counter() - t0_st, diff --git a/pyproject.toml b/pyproject.toml index 5f8aa1a..cb38cb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ dependencies = [ # jedi "jedi>=0.18.0,<0.20.0; python_version < '3.11'", "jedi<=0.19.2; python_version >= '3.11'", + # parso 0.8.5 is the first release shipping the Python 3.14 grammar; older + # resolutions make jedi reject every file in a 3.14 analysis env (#107) + "parso>=0.8.5", # msgpack "msgpack>=1.0.0,<1.0.7; python_version < '3.11'", "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'", diff --git a/test/test_env_interpreter.py b/test/test_env_interpreter.py new file mode 100644 index 0000000..68b1989 --- /dev/null +++ b/test/test_env_interpreter.py @@ -0,0 +1,156 @@ +"""Regression tests for #107: environment provisioning must prefer an +interpreter the installed jedi/parso stack can actually parse, and a run +where every module fails must not stay silent. + +parso ships one hardcoded grammar file per Python minor (grammar313.txt, +grammar314.txt, ...). If the provisioned analysis venv is newer than the +newest shipped grammar, jedi rejects every file and the symbol table comes +back empty while the process still exits 0. +""" +import logging +from pathlib import Path + +import pytest + +from codeanalyzer.core import Codeanalyzer + + +# ---------------------------------------------------------------------------------------------- +# The parso ceiling: derived from the shipped grammar files at runtime, never hardcoded. +# ---------------------------------------------------------------------------------------------- + + +def test_grammar_stems_parse_to_versions(): + got = Codeanalyzer._versions_from_grammar_stems( + ["grammar36", "grammar39", "grammar310", "grammar313", "grammar314"] + ) + assert got == [(3, 6), (3, 9), (3, 10), (3, 13), (3, 14)] + + +def test_malformed_grammar_stems_are_ignored(): + got = Codeanalyzer._versions_from_grammar_stems( + ["grammar", "grammarXY", "grammar3", "grammar312"] + ) + assert got == [(3, 12)] + + +def test_parso_ceiling_reflects_installed_parso(): + """The ceiling must be the max of the grammars parso actually ships — + on any env with parso >= 0.8.5 that is at least (3, 13).""" + ceiling = Codeanalyzer._parso_supported_ceiling() + assert ceiling is not None + assert ceiling >= (3, 13) + + +# ---------------------------------------------------------------------------------------------- +# Interpreter choice honors the ceiling. +# ---------------------------------------------------------------------------------------------- + + +def test_pick_supported_interpreter_prefers_newest_within_ceiling(): + candidates = [ + (Path("/opt/py315"), (3, 15)), + (Path("/opt/py313"), (3, 13)), + (Path("/opt/py311"), (3, 11)), + ] + assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 14)) == Path( + "/opt/py313" + ) + assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 10)) is None + + +def test_base_interpreter_swaps_out_unsupported_default(monkeypatch): + """If the default interpreter is newer than parso's ceiling, provisioning + must pick a supported one instead.""" + fake_default = Path("/opt/py399/bin/python3") + fake_supported = Path("/opt/py313/bin/python3") + + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, + "_interpreter_version", + staticmethod(lambda p: (3, 99) if p == fake_default else (3, 13)), + ) + monkeypatch.setattr( + Codeanalyzer, + "_find_supported_interpreter", + staticmethod(lambda ceiling: fake_supported), + ) + assert Codeanalyzer._get_base_interpreter() == fake_supported + + +def test_base_interpreter_keeps_supported_default(monkeypatch): + fake_default = Path("/opt/py312/bin/python3") + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 12)) + ) + assert Codeanalyzer._get_base_interpreter() == fake_default + + +def test_base_interpreter_falls_back_loudly_when_nothing_supported(monkeypatch, caplog): + fake_default = Path("/opt/py399/bin/python3") + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 99)) + ) + monkeypatch.setattr( + Codeanalyzer, "_find_supported_interpreter", staticmethod(lambda ceiling: None) + ) + # the codeanalyzer logger sets propagate=False (rich handler); caplog needs + # propagation to observe records + monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True) + with caplog.at_level(logging.WARNING, logger="codeanalyzer"): + assert Codeanalyzer._get_base_interpreter() == fake_default + assert any("parso" in r.getMessage() for r in caplog.records) + + +# ---------------------------------------------------------------------------------------------- +# A run where every module failed must be loud, not silently empty. +# ---------------------------------------------------------------------------------------------- + + +def test_all_files_failing_emits_an_error(tmp_path, monkeypatch, caplog): + proj = tmp_path / "proj" + proj.mkdir() + (proj / "a.py").write_text("def f():\n return 1\n", encoding="utf-8") + (proj / "b.py").write_text("def g():\n return 2\n", encoding="utf-8") + + from codeanalyzer.options.options import AnalysisOptions + from codeanalyzer.config import OutputFormat + from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder + + def boom(self, py_file): + raise RuntimeError("Python version 3.99 is currently not supported.") + + monkeypatch.setattr(SymbolTableBuilder, "build_pymodule_from_file", boom) + + opts = AnalysisOptions( + input=proj, output=None, format=OutputFormat.JSON, + skip_tests=True, no_venv=True, cache_dir=tmp_path / "cache", + rebuild_analysis=True, + ) + analyzer = Codeanalyzer(opts) + monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True) + with caplog.at_level(logging.ERROR, logger="codeanalyzer"): + table = analyzer._build_symbol_table(cached_symbol_table={}) + assert table == {} + assert any( + "every" in r.getMessage().lower() or "all " in r.getMessage().lower() + for r in caplog.records + ), "an empty symbol table from total per-file failure must be reported loudly"