From 858fa0acf47237bda439ee35a83577dd937305d2 Mon Sep 17 00:00:00 2001 From: stranske Date: Thu, 1 Jan 2026 20:44:35 +0000 Subject: [PATCH 1/3] feat: add local override for project modules in sync_test_dependencies Add support for .project_modules.txt file that consumer repos can use to specify additional first-party modules without modifying the synced script. This fixes recurring issues where repo-specific modules (like diff_holdings.py and embeddings.py in Manager-Database) get flagged as undeclared dependencies after workflow syncs. Usage: Create .project_modules.txt in repo root with one module per line. Lines starting with # are ignored as comments. --- scripts/sync_test_dependencies.py | 22 +++++++++- .../scripts/sync_test_dependencies.py | 42 +++++++++++++------ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/scripts/sync_test_dependencies.py b/scripts/sync_test_dependencies.py index 5ed337df0..b73df672d 100644 --- a/scripts/sync_test_dependencies.py +++ b/scripts/sync_test_dependencies.py @@ -18,6 +18,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] SRC_PATH = REPO_ROOT / "src" +LOCAL_MODULES_FILE = REPO_ROOT / ".project_modules.txt" if SRC_PATH.exists(): sys.path.insert(0, str(SRC_PATH)) @@ -176,9 +177,26 @@ def _detect_local_project_modules() -> set[str]: return detected +def _read_local_modules() -> set[str]: + """Read repo-specific module names from .project_modules.txt if it exists. + + This allows consumer repos to specify additional first-party modules + (like standalone .py files in root) without modifying this script. + One module name per line, comments start with #. + """ + if not LOCAL_MODULES_FILE.exists(): + return set() + modules: set[str] = set() + for line in LOCAL_MODULES_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + modules.add(line) + return modules + + def get_project_modules() -> set[str]: - """Return the full set of project modules (static + dynamically detected).""" - return _BASE_PROJECT_MODULES | _detect_local_project_modules() + """Return the full set of project modules (static + dynamically detected + local).""" + return _BASE_PROJECT_MODULES | _detect_local_project_modules() | _read_local_modules() # For backward compatibility - will be populated on first use diff --git a/templates/consumer-repo/scripts/sync_test_dependencies.py b/templates/consumer-repo/scripts/sync_test_dependencies.py index 9232f1394..b73df672d 100644 --- a/templates/consumer-repo/scripts/sync_test_dependencies.py +++ b/templates/consumer-repo/scripts/sync_test_dependencies.py @@ -18,6 +18,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] SRC_PATH = REPO_ROOT / "src" +LOCAL_MODULES_FILE = REPO_ROOT / ".project_modules.txt" if SRC_PATH.exists(): sys.path.insert(0, str(SRC_PATH)) @@ -75,6 +76,7 @@ "signal", "sitecustomize", "socket", + "sqlite3", "stat", "string", "struct", @@ -89,6 +91,7 @@ "unittest", "urllib", "uuid", + "venv", "warnings", "weakref", "xml", @@ -165,10 +168,8 @@ def _detect_local_project_modules() -> set[str]: continue # Check for packages (directories with __init__.py) - if item.is_dir(): - init_file = item / "__init__.py" - if init_file.exists(): - detected.add(item.name) + if item.is_dir() and (item / "__init__.py").exists(): + detected.add(item.name) # Check for standalone .py modules (but not in root .) elif source_dir != Path(".") and item.suffix == ".py": detected.add(item.stem) @@ -176,9 +177,26 @@ def _detect_local_project_modules() -> set[str]: return detected +def _read_local_modules() -> set[str]: + """Read repo-specific module names from .project_modules.txt if it exists. + + This allows consumer repos to specify additional first-party modules + (like standalone .py files in root) without modifying this script. + One module name per line, comments start with #. + """ + if not LOCAL_MODULES_FILE.exists(): + return set() + modules: set[str] = set() + for line in LOCAL_MODULES_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + modules.add(line) + return modules + + def get_project_modules() -> set[str]: - """Return the full set of project modules (static + dynamically detected).""" - return _BASE_PROJECT_MODULES | _detect_local_project_modules() + """Return the full set of project modules (static + dynamically detected + local).""" + return _BASE_PROJECT_MODULES | _detect_local_project_modules() | _read_local_modules() # For backward compatibility - will be populated on first use @@ -366,29 +384,29 @@ def main(argv: list[str] | None = None) -> int: missing = find_missing_dependencies() if not missing: - print("OK: All test dependencies are declared in pyproject.toml") + print("✅ All test dependencies are declared in pyproject.toml") return 0 - print(f"WARN: Found {len(missing)} undeclared dependencies:") + print(f"⚠️ Found {len(missing)} undeclared dependencies:") for dep in sorted(missing): print(f" - {dep}") if args.fix: added = add_dependencies_to_pyproject(missing, fix=True) if added: - print("\nOK: Added dependencies to [project.optional-dependencies.dev]") + print("\n✅ Added dependencies to [project.optional-dependencies.dev]") print("Please run: make lock") else: - print("\nINFO: Dependencies already declared in dev extra") + print("\nℹ️ Dependencies already declared in dev extra") return 0 if args.verify: - print("\nERROR: Run: python scripts/sync_test_dependencies.py --fix") + print("\n❌ Run: python scripts/sync_test_dependencies.py --fix") return 1 print("\nTo fix, run: python scripts/sync_test_dependencies.py --fix") return 0 -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - CLI entry point sys.exit(main()) From cbd967f6ee4178102463834fa3f6cfa788d9a356 Mon Sep 17 00:00:00 2001 From: stranske Date: Thu, 1 Jan 2026 21:07:31 +0000 Subject: [PATCH 2/3] fix: add validation and error handling to _read_local_modules Address Copilot review feedback: - Add try-except for OSError/UnicodeDecodeError when reading file - Validate module names are valid Python identifiers - Add comprehensive test coverage for _read_local_modules function Tests cover: - File not existing (returns empty set) - Valid module names parsed correctly - Comments and empty lines ignored - Whitespace stripped from entries - Invalid Python identifiers warned and skipped - File read errors handled gracefully - Integration with get_project_modules() --- scripts/sync_test_dependencies.py | 21 +++- .../scripts/sync_test_dependencies.py | 21 +++- tests/scripts/test_sync_test_dependencies.py | 110 ++++++++++++++++++ 3 files changed, 146 insertions(+), 6 deletions(-) diff --git a/scripts/sync_test_dependencies.py b/scripts/sync_test_dependencies.py index b73df672d..2c4beddba 100644 --- a/scripts/sync_test_dependencies.py +++ b/scripts/sync_test_dependencies.py @@ -187,10 +187,25 @@ def _read_local_modules() -> set[str]: if not LOCAL_MODULES_FILE.exists(): return set() modules: set[str] = set() - for line in LOCAL_MODULES_FILE.read_text(encoding="utf-8").splitlines(): + try: + content = LOCAL_MODULES_FILE.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + print( + f"Warning: could not read {LOCAL_MODULES_FILE}: {exc}", + file=sys.stderr, + ) + return set() + for line in content.splitlines(): line = line.strip() - if line and not line.startswith("#"): - modules.add(line) + if not line or line.startswith("#"): + continue + if not line.isidentifier(): + print( + f"Warning: ignoring invalid module name in {LOCAL_MODULES_FILE}: {line!r}", + file=sys.stderr, + ) + continue + modules.add(line) return modules diff --git a/templates/consumer-repo/scripts/sync_test_dependencies.py b/templates/consumer-repo/scripts/sync_test_dependencies.py index b73df672d..2c4beddba 100644 --- a/templates/consumer-repo/scripts/sync_test_dependencies.py +++ b/templates/consumer-repo/scripts/sync_test_dependencies.py @@ -187,10 +187,25 @@ def _read_local_modules() -> set[str]: if not LOCAL_MODULES_FILE.exists(): return set() modules: set[str] = set() - for line in LOCAL_MODULES_FILE.read_text(encoding="utf-8").splitlines(): + try: + content = LOCAL_MODULES_FILE.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + print( + f"Warning: could not read {LOCAL_MODULES_FILE}: {exc}", + file=sys.stderr, + ) + return set() + for line in content.splitlines(): line = line.strip() - if line and not line.startswith("#"): - modules.add(line) + if not line or line.startswith("#"): + continue + if not line.isidentifier(): + print( + f"Warning: ignoring invalid module name in {LOCAL_MODULES_FILE}: {line!r}", + file=sys.stderr, + ) + continue + modules.add(line) return modules diff --git a/tests/scripts/test_sync_test_dependencies.py b/tests/scripts/test_sync_test_dependencies.py index d6783a04a..a8970fa3a 100644 --- a/tests/scripts/test_sync_test_dependencies.py +++ b/tests/scripts/test_sync_test_dependencies.py @@ -374,3 +374,113 @@ def test_main_fix_mode_reports_already_declared( output = capsys.readouterr().out assert "Dependencies already declared in dev extra" in output + + +def test_read_local_modules_returns_empty_without_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test _read_local_modules returns empty set when file doesn't exist.""" + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", tmp_path / ".project_modules.txt") + + assert std._read_local_modules() == set() + + +def test_read_local_modules_reads_valid_module_names( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test _read_local_modules parses valid module names.""" + modules_file = tmp_path / ".project_modules.txt" + modules_file.write_text("diff_holdings\nembeddings\n", encoding="utf-8") + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + + result = std._read_local_modules() + + assert result == {"diff_holdings", "embeddings"} + + +def test_read_local_modules_ignores_comments_and_empty_lines( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test _read_local_modules skips comments and blank lines.""" + modules_file = tmp_path / ".project_modules.txt" + modules_file.write_text( + "# This is a comment\n" + "\n" + " # Indented comment \n" + "module_a\n" + " \n" + "module_b\n", + encoding="utf-8", + ) + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + + result = std._read_local_modules() + + assert result == {"module_a", "module_b"} + + +def test_read_local_modules_strips_whitespace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test _read_local_modules strips leading/trailing whitespace.""" + modules_file = tmp_path / ".project_modules.txt" + modules_file.write_text(" spaced_module \n\ttabbed_module\t\n", encoding="utf-8") + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + + result = std._read_local_modules() + + assert result == {"spaced_module", "tabbed_module"} + + +def test_read_local_modules_warns_on_invalid_names( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Test _read_local_modules warns about invalid Python identifiers.""" + modules_file = tmp_path / ".project_modules.txt" + modules_file.write_text( + "valid_module\n" + "123invalid\n" + "has-hyphen\n" + "has space\n" + "another_valid\n", + encoding="utf-8", + ) + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + + result = std._read_local_modules() + + assert result == {"valid_module", "another_valid"} + stderr = capsys.readouterr().err + assert "123invalid" in stderr + assert "has-hyphen" in stderr + assert "has space" in stderr + + +def test_read_local_modules_handles_read_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Test _read_local_modules gracefully handles file read errors.""" + modules_file = tmp_path / ".project_modules.txt" + # Create a directory with same name to cause read error + modules_file.mkdir() + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + + result = std._read_local_modules() + + assert result == set() + stderr = capsys.readouterr().err + assert "Warning" in stderr or "could not read" in stderr + + +def test_get_project_modules_includes_local_modules( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test get_project_modules includes modules from .project_modules.txt.""" + modules_file = tmp_path / ".project_modules.txt" + modules_file.write_text("custom_module\n", encoding="utf-8") + monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) + monkeypatch.chdir(tmp_path) + + result = std.get_project_modules() + + assert "custom_module" in result From de21cf1453b0038f4c9d67809958e47d21ff2876 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Jan 2026 21:08:03 +0000 Subject: [PATCH 3/3] chore(autofix): formatting/lint --- autofix_report_enriched.json | 2 +- tests/scripts/test_sync_test_dependencies.py | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/autofix_report_enriched.json b/autofix_report_enriched.json index 5f0169258..323183506 100644 --- a/autofix_report_enriched.json +++ b/autofix_report_enriched.json @@ -1 +1 @@ -{"changed": true, "classification": {"total": 0, "new": 0, "allowed": 0}, "timestamp": "2026-01-01T19:52:03Z", "files": ["scripts/sync_dev_dependencies.py", "templates/consumer-repo/scripts/sync_dev_dependencies.py", "tests/scripts/test_sync_dev_dependencies.py"]} \ No newline at end of file +{"changed": true, "classification": {"total": 0, "new": 0, "allowed": 0}, "timestamp": "2026-01-01T21:08:03Z", "files": ["tests/scripts/test_sync_test_dependencies.py"]} \ No newline at end of file diff --git a/tests/scripts/test_sync_test_dependencies.py b/tests/scripts/test_sync_test_dependencies.py index a8970fa3a..576def448 100644 --- a/tests/scripts/test_sync_test_dependencies.py +++ b/tests/scripts/test_sync_test_dependencies.py @@ -404,12 +404,7 @@ def test_read_local_modules_ignores_comments_and_empty_lines( """Test _read_local_modules skips comments and blank lines.""" modules_file = tmp_path / ".project_modules.txt" modules_file.write_text( - "# This is a comment\n" - "\n" - " # Indented comment \n" - "module_a\n" - " \n" - "module_b\n", + "# This is a comment\n" "\n" " # Indented comment \n" "module_a\n" " \n" "module_b\n", encoding="utf-8", ) monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file) @@ -438,11 +433,7 @@ def test_read_local_modules_warns_on_invalid_names( """Test _read_local_modules warns about invalid Python identifiers.""" modules_file = tmp_path / ".project_modules.txt" modules_file.write_text( - "valid_module\n" - "123invalid\n" - "has-hyphen\n" - "has space\n" - "another_valid\n", + "valid_module\n" "123invalid\n" "has-hyphen\n" "has space\n" "another_valid\n", encoding="utf-8", ) monkeypatch.setattr(std, "LOCAL_MODULES_FILE", modules_file)