diff --git a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js index 7f4304a64..7065993d9 100644 --- a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js +++ b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js @@ -17,6 +17,7 @@ const { buildPreamble, buildSourceContextRepairCommentBody, buildSourceContextResolvedCommentBody, + resolveExplicitNonIssueWorkflowSourceContext, resolveSourceContextRepairComment, resolveAgentType, stripPrTemplateContent, @@ -443,6 +444,47 @@ test('buildSourceContextResolvedCommentBody explains sync-campaign source contex assert.ok(result.includes('No linked GitHub issue is required')); }); +test('resolveExplicitNonIssueWorkflowSourceContext preserves explicit automation source despite stale issue preamble', () => { + const context = resolveExplicitNonIssueWorkflowSourceContext({ + body: [ + '', + '', + '> **Source:** Issue #1940', + '', + 'Closes #1940', + '', + '', + '', + '', + '', + '', + ].join('\n'), + }); + + assert.deepEqual(context, { + sourceType: 'automation_run', + issueNumber: null, + sourceRef: 'workflows-system-review-2 slice 112', + lifecycle: 'implementation_slice', + automation: 'optional_local_or_remote', + isKnown: true, + isValid: true, + isExplicit: true, + requiresIssue: false, + }); +}); + +test('resolveExplicitNonIssueWorkflowSourceContext ignores source issue markers', () => { + const context = resolveExplicitNonIssueWorkflowSourceContext({ + body: [ + '', + '', + ].join('\n'), + }); + + assert.equal(context, null); +}); + test('resolveSourceContextRepairComment updates an existing warning once', async () => { const calls = { update: 0, body: '' }; const github = { diff --git a/.github/scripts/agents_pr_meta_update_body.js b/.github/scripts/agents_pr_meta_update_body.js index 403e76803..89c24c55d 100644 --- a/.github/scripts/agents_pr_meta_update_body.js +++ b/.github/scripts/agents_pr_meta_update_body.js @@ -17,7 +17,10 @@ const childProcess = require('child_process'); const { ensureRateLimitWrapped } = require('./github-rate-limited-wrapper.js'); const { formatSourceContextForLog, + normalizeSourceType, resolvePrSourceContext, + SOURCE_TYPES, + VALID_SOURCE_TYPES, } = require('./source_context.js'); class RateLimitError extends Error { @@ -987,6 +990,39 @@ function buildSourceContextResolvedCommentBody(prNumber, sourceContext) { ].join('\n'); } +function parseHtmlMarker(body, name) { + const pattern = new RegExp(``, 'i'); + const match = String(body || '').match(pattern); + return match ? String(match[1] || '').trim() : ''; +} + +function resolveExplicitNonIssueWorkflowSourceContext(pr = {}) { + const body = String(pr.body || ''); + const sourceType = normalizeSourceType(parseHtmlMarker(body, 'workflow-source')); + if ( + !VALID_SOURCE_TYPES.has(sourceType) + || sourceType === SOURCE_TYPES.GITHUB_ISSUE + || sourceType === SOURCE_TYPES.UNKNOWN + ) { + return null; + } + + const sourceRef = parseHtmlMarker(body, 'workflow-source-ref'); + const lifecycle = parseHtmlMarker(body, 'workflow-lifecycle'); + const automation = parseHtmlMarker(body, 'workflow-automation'); + return { + sourceType, + issueNumber: null, + sourceRef, + lifecycle, + automation, + isKnown: true, + isValid: true, + isExplicit: true, + requiresIssue: false, + }; +} + async function resolveSourceContextRepairComment({ github, owner, @@ -1320,6 +1356,32 @@ async function run({github: rawGithub, context, core, inputs}) { return; } + const explicitNonIssueSourceContext = resolveExplicitNonIssueWorkflowSourceContext(pr); + if (explicitNonIssueSourceContext) { + core.info( + `PR #${pr.number} has explicit non-issue workflow source context (${formatSourceContextForLog(explicitNonIssueSourceContext)}); skipping issue-sourced body sync.`, + ); + try { + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: pr.number, + }); + await resolveSourceContextRepairComment({ + github, + owner, + repo, + prNumber: pr.number, + comments, + sourceContext: explicitNonIssueSourceContext, + core, + }); + } catch (error) { + core.warning(`Failed to resolve workflow source repair comment: ${error.message}`); + } + return; + } + const issueNumber = extractIssueNumberFromPull(pr); const sourceContext = resolvePrSourceContext(pr); if (!issueNumber) { @@ -1566,6 +1628,7 @@ module.exports = { buildPreamble, buildSourceContextRepairCommentBody, buildSourceContextResolvedCommentBody, + resolveExplicitNonIssueWorkflowSourceContext, resolveSourceContextRepairComment, isCampaignIssue, buildStatusBlock, diff --git a/.github/workflows/maint-46-post-ci.yml b/.github/workflows/maint-46-post-ci.yml index 6439b10ed..60368429e 100644 --- a/.github/workflows/maint-46-post-ci.yml +++ b/.github/workflows/maint-46-post-ci.yml @@ -210,7 +210,7 @@ jobs: if: ${{ steps.gate_guard.outputs.recover == 'true' }} id: render run: | - python tools/post_ci_summary.py + python -m tools.post_ci_summary env: RUNS_JSON: ${{ steps.discover.outputs.runs }} HEAD_SHA: ${{ steps.discover.outputs.head_sha }} diff --git a/.github/workflows/pr-00-gate.yml b/.github/workflows/pr-00-gate.yml index ea27fe37e..be54447cd 100644 --- a/.github/workflows/pr-00-gate.yml +++ b/.github/workflows/pr-00-gate.yml @@ -582,7 +582,7 @@ jobs: - name: Prepare summary body id: summary_body run: | - python tools/post_ci_summary.py + python -m tools.post_ci_summary env: PYTHONPATH: ${{ github.workspace }} RUNS_JSON: ${{ steps.gather.outputs.runs || '[]' }} diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index a374643c1..923fad967 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -18,8 +18,13 @@ _DEFAULT_OUTPUT = "agent-metrics-summary.md" _DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_DOWNLOAD_MANIFEST_PATH = "artifacts/metric-artifact-download-manifest.json" +_DEFAULT_ARTIFACT_SELECTION_PATH = "artifacts/metric-artifacts-selection.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" +_TERMINAL_ARTIFACT_FAMILIES = ( + "review-thread-terminal-disposition", + "verifier-terminal-disposition", +) _EXACT_ARTIFACT_FAMILIES = { "keepalive-metrics", "agents-autofix-metrics", @@ -1022,10 +1027,142 @@ def _read_artifact_download_contract(manifest_path: Path) -> dict[str, Any] | No return _artifact_download_contract(manifest, manifest_path) +def _compact_artifact_ref(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + return { + "id": value.get("id"), + "name": value.get("name") or "", + "created_at": value.get("created_at") or "", + "updated_at": value.get("updated_at") or "", + } + + +def _artifact_selection_contract(selection: dict[str, Any], selection_path: Path) -> dict[str, Any]: + statuses: list[dict[str, Any]] = [] + priority_statuses = selection.get("priority_family_statuses") + if isinstance(priority_statuses, list): + for item in priority_statuses: + if not isinstance(item, dict): + continue + family = str(item.get("family") or "") + if family not in _TERMINAL_ARTIFACT_FAMILIES: + continue + statuses.append( + { + "family": family, + "status": str(item.get("status") or "unknown"), + "candidate_count": _safe_int(item.get("candidate_count")) or 0, + "selected_count": _safe_int(item.get("selected_count")) or 0, + "latest_candidate": _compact_artifact_ref(item.get("latest_candidate")), + "selected_artifact": _compact_artifact_ref(item.get("selected_artifact")), + } + ) + + seen = {item["family"] for item in statuses} + candidate_counts = selection.get("candidate_family_counts") + selected_counts = selection.get("selected_family_counts") + candidate_counts = candidate_counts if isinstance(candidate_counts, dict) else {} + selected_counts = selected_counts if isinstance(selected_counts, dict) else {} + for family in _TERMINAL_ARTIFACT_FAMILIES: + if family in seen: + continue + candidate_count = _safe_int(candidate_counts.get(family)) or 0 + selected_count = _safe_int(selected_counts.get(family)) or 0 + status = "selected" if selected_count else "missing" + statuses.append( + { + "family": family, + "status": status, + "candidate_count": candidate_count, + "selected_count": selected_count, + "latest_candidate": None, + "selected_artifact": None, + } + ) + + statuses.sort(key=lambda item: _TERMINAL_ARTIFACT_FAMILIES.index(item["family"])) + missing_terminal = [ + item["family"] + for item in statuses + if item["status"] == "missing" or item["selected_count"] <= 0 + ] + return { + "schema": selection.get("schema") or "unknown", + "path": selection_path.as_posix(), + "status": selection.get("status") or "unknown", + "selected_count": _safe_int(selection.get("selected_count")) or 0, + "candidate_count": _safe_int(selection.get("candidate_count")) or 0, + "missing_priority_families": list(selection.get("missing_priority_families") or []), + "terminal_artifact_families": statuses, + "missing_terminal_artifact_families": missing_terminal, + } + + +def _read_artifact_selection_contract(selection_path: Path) -> dict[str, Any] | None: + if not selection_path.exists(): + return None + try: + selection = json.loads(selection_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return { + "schema": "workflows-weekly-metrics-artifact-selection/v1", + "path": selection_path.as_posix(), + "status": "error", + "error_message": str(exc), + "selected_count": 0, + "candidate_count": 0, + "missing_priority_families": [], + "terminal_artifact_families": [], + "missing_terminal_artifact_families": list(_TERMINAL_ARTIFACT_FAMILIES), + } + if not isinstance(selection, dict): + return { + "schema": "workflows-weekly-metrics-artifact-selection/v1", + "path": selection_path.as_posix(), + "status": "error", + "error_message": "selection-not-object", + "selected_count": 0, + "candidate_count": 0, + "missing_priority_families": [], + "terminal_artifact_families": [], + "missing_terminal_artifact_families": list(_TERMINAL_ARTIFACT_FAMILIES), + } + return _artifact_selection_contract(selection, selection_path) + + +def _format_terminal_artifact_statuses(artifact_selection: dict[str, Any] | None) -> str: + if not artifact_selection: + return "n/a" + statuses = artifact_selection.get("terminal_artifact_families") + if not isinstance(statuses, list) or not statuses: + return "n/a" + parts = [] + for item in statuses: + if not isinstance(item, dict): + continue + family = item.get("family") or "unknown" + status = item.get("status") or "unknown" + selected = item.get("selected_count") or 0 + candidates = item.get("candidate_count") or 0 + parts.append(f"{family}: {status} ({selected}/{candidates})") + return ", ".join(parts) if parts else "n/a" + + +def _format_missing_terminal_artifact_families( + artifact_selection: dict[str, Any] | None, +) -> str: + if not artifact_selection: + return "n/a" + missing = artifact_selection.get("missing_terminal_artifact_families") or [] + return ", ".join(str(item) for item in missing) if missing else "none" + + def build_summary( entries: list[dict[str, Any]], errors: int, parse_error_details: list[ParseErrorDetail] | None = None, + artifact_selection: dict[str, Any] | None = None, ) -> str: buckets = _bucket_entries(entries) timestamps: list[_dt.datetime] = [] @@ -1100,6 +1237,11 @@ def build_summary( f"- Terminal disposition records: {verifier['terminal_records']}", f"- Terminal dispositions: {_format_counter(verifier['terminal_dispositions'])}", f"- Terminal disposition sources: {_format_counter(verifier['terminal_sources'])}", + f"- Terminal artifact families: {_format_terminal_artifact_statuses(artifact_selection)}", + ( + "- Missing terminal artifact families: " + f"{_format_missing_terminal_artifact_families(artifact_selection)}" + ), f"- Verifier follow-up ledger records: {verifier['ledger_records']}", f"- Verifier follow-up ledger dispositions: {_format_counter(verifier['ledger_dispositions'])}", f"- Verifier follow-up ledger PRs: {verifier['ledger_prs']}", @@ -1197,6 +1339,7 @@ def build_summary_contract( entries: list[dict[str, Any]], parse_error_details: list[ParseErrorDetail], artifact_downloads: dict[str, Any] | None = None, + artifact_selection: dict[str, Any] | None = None, ) -> dict[str, Any]: entry_buckets = _bucket_entries(entries) buckets: dict[str, int] = Counter( @@ -1223,6 +1366,8 @@ def build_summary_contract( } if artifact_downloads is not None: contract["artifact_downloads"] = artifact_downloads + if artifact_selection is not None: + contract["artifact_selection"] = artifact_selection if timestamps: contract["range"] = { "earliest": min(timestamps).isoformat().replace("+00:00", "Z"), @@ -1241,6 +1386,10 @@ def main() -> int: os.environ.get("METRICS_ARTIFACT_DOWNLOAD_MANIFEST_JSON", _DEFAULT_DOWNLOAD_MANIFEST_PATH) ) artifact_downloads = _read_artifact_download_contract(download_manifest_path) + artifact_selection_path = Path( + os.environ.get("METRICS_ARTIFACT_SELECTION_JSON", _DEFAULT_ARTIFACT_SELECTION_PATH) + ) + artifact_selection = _read_artifact_selection_contract(artifact_selection_path) files = _gather_metrics_files(metrics_paths, metrics_dir) if not files: @@ -1249,7 +1398,7 @@ def main() -> int: output_json_path.parent.mkdir(parents=True, exist_ok=True) output_json_path.write_text( json.dumps( - build_summary_contract([], [], artifact_downloads), + build_summary_contract([], [], artifact_downloads, artifact_selection), indent=2, sort_keys=True, ) @@ -1260,8 +1409,18 @@ def main() -> int: return 0 entries, parse_error_details = _read_ndjson(files) - summary = build_summary(entries, _parse_error_count(parse_error_details), parse_error_details) - summary_contract = build_summary_contract(entries, parse_error_details, artifact_downloads) + summary = build_summary( + entries, + _parse_error_count(parse_error_details), + parse_error_details, + artifact_selection, + ) + summary_contract = build_summary_contract( + entries, + parse_error_details, + artifact_downloads, + artifact_selection, + ) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(summary, encoding="utf-8") diff --git a/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js b/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js index 403e76803..89c24c55d 100644 --- a/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js +++ b/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js @@ -17,7 +17,10 @@ const childProcess = require('child_process'); const { ensureRateLimitWrapped } = require('./github-rate-limited-wrapper.js'); const { formatSourceContextForLog, + normalizeSourceType, resolvePrSourceContext, + SOURCE_TYPES, + VALID_SOURCE_TYPES, } = require('./source_context.js'); class RateLimitError extends Error { @@ -987,6 +990,39 @@ function buildSourceContextResolvedCommentBody(prNumber, sourceContext) { ].join('\n'); } +function parseHtmlMarker(body, name) { + const pattern = new RegExp(``, 'i'); + const match = String(body || '').match(pattern); + return match ? String(match[1] || '').trim() : ''; +} + +function resolveExplicitNonIssueWorkflowSourceContext(pr = {}) { + const body = String(pr.body || ''); + const sourceType = normalizeSourceType(parseHtmlMarker(body, 'workflow-source')); + if ( + !VALID_SOURCE_TYPES.has(sourceType) + || sourceType === SOURCE_TYPES.GITHUB_ISSUE + || sourceType === SOURCE_TYPES.UNKNOWN + ) { + return null; + } + + const sourceRef = parseHtmlMarker(body, 'workflow-source-ref'); + const lifecycle = parseHtmlMarker(body, 'workflow-lifecycle'); + const automation = parseHtmlMarker(body, 'workflow-automation'); + return { + sourceType, + issueNumber: null, + sourceRef, + lifecycle, + automation, + isKnown: true, + isValid: true, + isExplicit: true, + requiresIssue: false, + }; +} + async function resolveSourceContextRepairComment({ github, owner, @@ -1320,6 +1356,32 @@ async function run({github: rawGithub, context, core, inputs}) { return; } + const explicitNonIssueSourceContext = resolveExplicitNonIssueWorkflowSourceContext(pr); + if (explicitNonIssueSourceContext) { + core.info( + `PR #${pr.number} has explicit non-issue workflow source context (${formatSourceContextForLog(explicitNonIssueSourceContext)}); skipping issue-sourced body sync.`, + ); + try { + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: pr.number, + }); + await resolveSourceContextRepairComment({ + github, + owner, + repo, + prNumber: pr.number, + comments, + sourceContext: explicitNonIssueSourceContext, + core, + }); + } catch (error) { + core.warning(`Failed to resolve workflow source repair comment: ${error.message}`); + } + return; + } + const issueNumber = extractIssueNumberFromPull(pr); const sourceContext = resolvePrSourceContext(pr); if (!issueNumber) { @@ -1566,6 +1628,7 @@ module.exports = { buildPreamble, buildSourceContextRepairCommentBody, buildSourceContextResolvedCommentBody, + resolveExplicitNonIssueWorkflowSourceContext, resolveSourceContextRepairComment, isCampaignIssue, buildStatusBlock, diff --git a/templates/consumer-repo/.github/workflows/pr-00-gate.yml b/templates/consumer-repo/.github/workflows/pr-00-gate.yml index a97247a91..eb46da199 100644 --- a/templates/consumer-repo/.github/workflows/pr-00-gate.yml +++ b/templates/consumer-repo/.github/workflows/pr-00-gate.yml @@ -581,7 +581,7 @@ jobs: - name: Prepare summary body id: summary_body run: | - python tools/post_ci_summary.py + python -m tools.post_ci_summary env: PYTHONPATH: ${{ github.workspace }} RUNS_JSON: ${{ steps.gather.outputs.runs || '[]' }} diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index a374643c1..923fad967 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -18,8 +18,13 @@ _DEFAULT_OUTPUT = "agent-metrics-summary.md" _DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_DOWNLOAD_MANIFEST_PATH = "artifacts/metric-artifact-download-manifest.json" +_DEFAULT_ARTIFACT_SELECTION_PATH = "artifacts/metric-artifacts-selection.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" +_TERMINAL_ARTIFACT_FAMILIES = ( + "review-thread-terminal-disposition", + "verifier-terminal-disposition", +) _EXACT_ARTIFACT_FAMILIES = { "keepalive-metrics", "agents-autofix-metrics", @@ -1022,10 +1027,142 @@ def _read_artifact_download_contract(manifest_path: Path) -> dict[str, Any] | No return _artifact_download_contract(manifest, manifest_path) +def _compact_artifact_ref(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + return { + "id": value.get("id"), + "name": value.get("name") or "", + "created_at": value.get("created_at") or "", + "updated_at": value.get("updated_at") or "", + } + + +def _artifact_selection_contract(selection: dict[str, Any], selection_path: Path) -> dict[str, Any]: + statuses: list[dict[str, Any]] = [] + priority_statuses = selection.get("priority_family_statuses") + if isinstance(priority_statuses, list): + for item in priority_statuses: + if not isinstance(item, dict): + continue + family = str(item.get("family") or "") + if family not in _TERMINAL_ARTIFACT_FAMILIES: + continue + statuses.append( + { + "family": family, + "status": str(item.get("status") or "unknown"), + "candidate_count": _safe_int(item.get("candidate_count")) or 0, + "selected_count": _safe_int(item.get("selected_count")) or 0, + "latest_candidate": _compact_artifact_ref(item.get("latest_candidate")), + "selected_artifact": _compact_artifact_ref(item.get("selected_artifact")), + } + ) + + seen = {item["family"] for item in statuses} + candidate_counts = selection.get("candidate_family_counts") + selected_counts = selection.get("selected_family_counts") + candidate_counts = candidate_counts if isinstance(candidate_counts, dict) else {} + selected_counts = selected_counts if isinstance(selected_counts, dict) else {} + for family in _TERMINAL_ARTIFACT_FAMILIES: + if family in seen: + continue + candidate_count = _safe_int(candidate_counts.get(family)) or 0 + selected_count = _safe_int(selected_counts.get(family)) or 0 + status = "selected" if selected_count else "missing" + statuses.append( + { + "family": family, + "status": status, + "candidate_count": candidate_count, + "selected_count": selected_count, + "latest_candidate": None, + "selected_artifact": None, + } + ) + + statuses.sort(key=lambda item: _TERMINAL_ARTIFACT_FAMILIES.index(item["family"])) + missing_terminal = [ + item["family"] + for item in statuses + if item["status"] == "missing" or item["selected_count"] <= 0 + ] + return { + "schema": selection.get("schema") or "unknown", + "path": selection_path.as_posix(), + "status": selection.get("status") or "unknown", + "selected_count": _safe_int(selection.get("selected_count")) or 0, + "candidate_count": _safe_int(selection.get("candidate_count")) or 0, + "missing_priority_families": list(selection.get("missing_priority_families") or []), + "terminal_artifact_families": statuses, + "missing_terminal_artifact_families": missing_terminal, + } + + +def _read_artifact_selection_contract(selection_path: Path) -> dict[str, Any] | None: + if not selection_path.exists(): + return None + try: + selection = json.loads(selection_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return { + "schema": "workflows-weekly-metrics-artifact-selection/v1", + "path": selection_path.as_posix(), + "status": "error", + "error_message": str(exc), + "selected_count": 0, + "candidate_count": 0, + "missing_priority_families": [], + "terminal_artifact_families": [], + "missing_terminal_artifact_families": list(_TERMINAL_ARTIFACT_FAMILIES), + } + if not isinstance(selection, dict): + return { + "schema": "workflows-weekly-metrics-artifact-selection/v1", + "path": selection_path.as_posix(), + "status": "error", + "error_message": "selection-not-object", + "selected_count": 0, + "candidate_count": 0, + "missing_priority_families": [], + "terminal_artifact_families": [], + "missing_terminal_artifact_families": list(_TERMINAL_ARTIFACT_FAMILIES), + } + return _artifact_selection_contract(selection, selection_path) + + +def _format_terminal_artifact_statuses(artifact_selection: dict[str, Any] | None) -> str: + if not artifact_selection: + return "n/a" + statuses = artifact_selection.get("terminal_artifact_families") + if not isinstance(statuses, list) or not statuses: + return "n/a" + parts = [] + for item in statuses: + if not isinstance(item, dict): + continue + family = item.get("family") or "unknown" + status = item.get("status") or "unknown" + selected = item.get("selected_count") or 0 + candidates = item.get("candidate_count") or 0 + parts.append(f"{family}: {status} ({selected}/{candidates})") + return ", ".join(parts) if parts else "n/a" + + +def _format_missing_terminal_artifact_families( + artifact_selection: dict[str, Any] | None, +) -> str: + if not artifact_selection: + return "n/a" + missing = artifact_selection.get("missing_terminal_artifact_families") or [] + return ", ".join(str(item) for item in missing) if missing else "none" + + def build_summary( entries: list[dict[str, Any]], errors: int, parse_error_details: list[ParseErrorDetail] | None = None, + artifact_selection: dict[str, Any] | None = None, ) -> str: buckets = _bucket_entries(entries) timestamps: list[_dt.datetime] = [] @@ -1100,6 +1237,11 @@ def build_summary( f"- Terminal disposition records: {verifier['terminal_records']}", f"- Terminal dispositions: {_format_counter(verifier['terminal_dispositions'])}", f"- Terminal disposition sources: {_format_counter(verifier['terminal_sources'])}", + f"- Terminal artifact families: {_format_terminal_artifact_statuses(artifact_selection)}", + ( + "- Missing terminal artifact families: " + f"{_format_missing_terminal_artifact_families(artifact_selection)}" + ), f"- Verifier follow-up ledger records: {verifier['ledger_records']}", f"- Verifier follow-up ledger dispositions: {_format_counter(verifier['ledger_dispositions'])}", f"- Verifier follow-up ledger PRs: {verifier['ledger_prs']}", @@ -1197,6 +1339,7 @@ def build_summary_contract( entries: list[dict[str, Any]], parse_error_details: list[ParseErrorDetail], artifact_downloads: dict[str, Any] | None = None, + artifact_selection: dict[str, Any] | None = None, ) -> dict[str, Any]: entry_buckets = _bucket_entries(entries) buckets: dict[str, int] = Counter( @@ -1223,6 +1366,8 @@ def build_summary_contract( } if artifact_downloads is not None: contract["artifact_downloads"] = artifact_downloads + if artifact_selection is not None: + contract["artifact_selection"] = artifact_selection if timestamps: contract["range"] = { "earliest": min(timestamps).isoformat().replace("+00:00", "Z"), @@ -1241,6 +1386,10 @@ def main() -> int: os.environ.get("METRICS_ARTIFACT_DOWNLOAD_MANIFEST_JSON", _DEFAULT_DOWNLOAD_MANIFEST_PATH) ) artifact_downloads = _read_artifact_download_contract(download_manifest_path) + artifact_selection_path = Path( + os.environ.get("METRICS_ARTIFACT_SELECTION_JSON", _DEFAULT_ARTIFACT_SELECTION_PATH) + ) + artifact_selection = _read_artifact_selection_contract(artifact_selection_path) files = _gather_metrics_files(metrics_paths, metrics_dir) if not files: @@ -1249,7 +1398,7 @@ def main() -> int: output_json_path.parent.mkdir(parents=True, exist_ok=True) output_json_path.write_text( json.dumps( - build_summary_contract([], [], artifact_downloads), + build_summary_contract([], [], artifact_downloads, artifact_selection), indent=2, sort_keys=True, ) @@ -1260,8 +1409,18 @@ def main() -> int: return 0 entries, parse_error_details = _read_ndjson(files) - summary = build_summary(entries, _parse_error_count(parse_error_details), parse_error_details) - summary_contract = build_summary_contract(entries, parse_error_details, artifact_downloads) + summary = build_summary( + entries, + _parse_error_count(parse_error_details), + parse_error_details, + artifact_selection, + ) + summary_contract = build_summary_contract( + entries, + parse_error_details, + artifact_downloads, + artifact_selection, + ) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(summary, encoding="utf-8") diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index cf12006d1..894354553 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -213,6 +213,86 @@ def test_main_writes_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> monkeypatch.delenv("OUTPUT_JSON_PATH", raising=False) +def test_main_exposes_terminal_artifact_family_selection( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + metrics_path = tmp_path / "terminal.ndjson" + output_path = tmp_path / "summary.md" + output_json_path = tmp_path / "summary.json" + selection_path = tmp_path / "metric-artifacts-selection.json" + + _write_ndjson( + metrics_path, + [ + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "review-thread-terminal-disposition", + "source_type": "review-thread", + "source_id": "1927", + "disposition": "wrapper-skipped", + } + ], + ) + selection_path.write_text( + json.dumps( + { + "schema": "workflows-weekly-metrics-artifact-selection/v1", + "status": "pass", + "candidate_count": 60, + "selected_count": 20, + "missing_priority_families": ["verifier-terminal-disposition"], + "priority_family_statuses": [ + { + "family": "verifier-terminal-disposition", + "status": "missing", + "candidate_count": 0, + "selected_count": 0, + "latest_candidate": None, + "selected_artifact": None, + }, + { + "family": "review-thread-terminal-disposition", + "status": "selected", + "candidate_count": 60, + "selected_count": 20, + "latest_candidate": { + "id": 6651095636, + "name": "review-thread-terminal-disposition-24969526334", + }, + "selected_artifact": { + "id": 6651095636, + "name": "review-thread-terminal-disposition-24969526334", + }, + }, + ], + } + ), + encoding="utf-8", + ) + + monkeypatch.setenv("METRICS_PATHS", str(metrics_path)) + monkeypatch.setenv("OUTPUT_PATH", str(output_path)) + monkeypatch.setenv("OUTPUT_JSON_PATH", str(output_json_path)) + monkeypatch.setenv("METRICS_ARTIFACT_SELECTION_JSON", str(selection_path)) + + assert aggregate_agent_metrics.main() == 0 + + summary = output_path.read_text(encoding="utf-8") + assert ( + "Terminal artifact families: review-thread-terminal-disposition: selected (20/60), " + "verifier-terminal-disposition: missing (0/0)" + ) in summary + assert "Missing terminal artifact families: verifier-terminal-disposition" in summary + + summary_json = json.loads(output_json_path.read_text(encoding="utf-8")) + artifact_selection = summary_json["artifact_selection"] + assert artifact_selection["missing_terminal_artifact_families"] == [ + "verifier-terminal-disposition" + ] + assert artifact_selection["terminal_artifact_families"][1]["status"] == "missing" + + def test_parse_timestamp_variants() -> None: epoch = aggregate_agent_metrics._parse_timestamp(0) assert epoch is not None diff --git a/tests/workflows/test_maint46_post_ci_sparse_checkout.py b/tests/workflows/test_maint46_post_ci_sparse_checkout.py index c44b2d6fb..97e9a6eae 100644 --- a/tests/workflows/test_maint46_post_ci_sparse_checkout.py +++ b/tests/workflows/test_maint46_post_ci_sparse_checkout.py @@ -16,6 +16,17 @@ def test_maint46_sparse_checkout_includes_post_ci_import_dependencies(): assert "tools/ci_failure_triage.py" in sparse_checkout +def test_maint46_runs_post_ci_summary_as_importable_module(): + workflow = yaml.safe_load( + Path(".github/workflows/maint-46-post-ci.yml").read_text(encoding="utf-8") + ) + steps = workflow["jobs"]["summary"]["steps"] + render = next(step for step in steps if step.get("name") == "Build summary body") + + assert "python -m tools.post_ci_summary" in render["run"] + assert "python tools/post_ci_summary.py" not in render["run"] + + def test_maint46_gate_artifact_download_fails_open_to_metadata_summary(): workflow = yaml.safe_load( Path(".github/workflows/maint-46-post-ci.yml").read_text(encoding="utf-8")