Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,29 @@ test('compactMarkerPayload exposes the current drift checkpoint', () => {
assert.equal(payload.follow_up.workflow, 'maint-68-sync-consumer-repos.yml');
});

test('formatIssueComment suppresses covered-state noise', () => {
assert.equal(formatIssueComment({ ...report, status: 'covered' }), '');
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.

test('formatIssueBody renders coverage details while preserving actionable output', () => {
const covered = formatIssueBody({
...report,
status: 'covered',
sync_remediation: {
...report.sync_remediation,
expected_branch: 'sync/workflows-aaaaaaaaaaaa',
coverage_lease_hours: 36,
},
});
assert.match(covered, /covered by current, unexpired compiler-plan sync PRs/);
assert.match(covered, /Current plan branch: `sync\/workflows-aaaaaaaaaaaa`/);
assert.match(covered, /Coverage lease: 36 hours/);

const actionable = formatIssueBody(report);
assert.match(actionable, /One or more consumer repos have actionable drift/);
assert.doesNotMatch(actionable, /Current plan branch:/);
});

test('mergeIssueBody refreshes generated issue bodies', () => {
const oldBody = formatIssueBody({
counts: { drift: 1, missing: 0, errors: 0, obsolete: 0 },
Expand Down
16 changes: 15 additions & 1 deletion .github/scripts/consumer_sync_drift_issue_body.js
Original file line number Diff line number Diff line change
Expand Up @@ -122,12 +122,16 @@ function formatIssueBody(report, options = {}) {
const runLink = runUrl && runNumber ? `[Run #${runNumber}](${runUrl})` : runUrl || 'current run';
const openSyncPrs = formatOpenSyncPrs(report);

const remediation = report && report.sync_remediation ? report.sync_remediation : {};
const isCovered = report && report.status === 'covered';
const lines = [
'## Consumer Repo Drift Detected',
'',
'> **Durable tracker** — see [`docs/ops/DURABLE_TRACKING_ISSUES.md`](https://github.com/stranske/Workflows/blob/main/docs/ops/DURABLE_TRACKING_ISSUES.md). The body below is regenerated each cycle by `health-68-consumer-sync-drift.yml`; auto-resolves on the next clean run.',
'',
'One or more consumer repos have drifted from the Workflows templates or manifest entries.',
isCovered
? 'Detected drift is covered by current, unexpired compiler-plan sync PRs; no tracker comment is needed.'
: 'One or more consumer repos have actionable drift from the Workflows templates or manifest entries.',
'',
`**Check Details:** ${runLink}`,
`**Counts:** ${countsLine(report)}`,
Expand All @@ -144,6 +148,13 @@ function formatIssueBody(report, options = {}) {
'- Close this issue when Health 68 passes.',
'',
];
if (remediation.expected_branch) {
lines.splice(lines.indexOf('### Required Actions'), 0,
'### Remediation state',
`- Current plan branch: \`${remediation.expected_branch}\``,
`- Coverage lease: ${remediation.coverage_lease_hours || 0} hours`,
'');
}
if (openSyncPrs.length > 0) {
lines.push(
'### Open sync PRs',
Expand Down Expand Up @@ -175,6 +186,9 @@ function mergeIssueBody(existingBody, report, options = {}) {
}

function formatIssueComment(report, options = {}) {
if (report && report.status === 'covered') {
return '';
Comment thread
stranske marked this conversation as resolved.
}
Comment thread
stranske marked this conversation as resolved.
const runUrl = options.runUrl || '';
const runNumber = options.runNumber || '';
const runLink = runUrl && runNumber ? `[run #${runNumber}](${runUrl})` : runUrl || 'latest run';
Expand Down
23 changes: 16 additions & 7 deletions .github/workflows/health-68-consumer-sync-drift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ on:
- '.github/scripts/**'
- 'scripts/**'
- 'tools/**'
schedule:
- cron: '10 5 * * *' # Daily at 5:10 UTC
workflow_run:
workflows: [Merge Sync PRs]
types: [completed]
workflow_dispatch:
inputs:
repos:
Expand All @@ -32,6 +33,9 @@ concurrency:
jobs:
check-drift:
name: Validate consumer repo drift
if: >-
github.event_name != 'workflow_run' ||
(github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main')
runs-on: ubuntu-latest
steps:
- name: Checkout Workflows repo
Expand Down Expand Up @@ -183,15 +187,20 @@ jobs:
return;
}
console.log(`Refreshed drift issue body for #${tracker.number}`);
const comment = formatIssueComment(report, {
runUrl,
runId: process.env.GITHUB_RUN_ID,
runNumber: process.env.GITHUB_RUN_NUMBER,
});
if (!comment.trim()) {
console.log(`Skipped empty comment for existing issue #${tracker.number}`);
return;
}
const commentResult = await skipOnRateLimit('comment', () => appendTrackerComment({
github,
context,
tracker,
comment: formatIssueComment(report, {
runUrl,
runId: process.env.GITHUB_RUN_ID,
runNumber: process.env.GITHUB_RUN_NUMBER,
}),
comment,
core,
withRetry,
}));
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/maint-71-merge-sync-prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ name: Merge Sync PRs
on:
schedule:
# Daily janitor pass: auto-merge ready sync PRs, close superseded ones, and prune
# leftover branches without waiting for a manual dispatch. Runs just after Health 68's
# drift scan (cron '10 5') so it acts on a fresh picture. A schedule event carries no
# leftover branches without waiting for a manual dispatch. Health 68 runs from this
# workflow's successful completion, so it evaluates the post-janitor state. A schedule event carries no
# inputs, so the script's parseBooleanInput defaults apply unchanged:
# auto_merge=true, dry_run=false, cleanup_branches=true, repos=all.
- cron: '30 5 * * *'
Expand Down
11 changes: 11 additions & 0 deletions docs/ops/CONSUMER_REPO_MAINTENANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,17 @@ The list of registered repos lives in that workflow (env var
`REGISTERED_CONSUMER_REPOS`). Avoid duplicating the list here; it changes over time and
the workflow is the source of truth.

### Drift coverage states

Scheduled Health 68 runs are triggered only after a successful Maint 71 janitor; push and
manual runs are intentionally immediate. It classifies each
consumer as `converged`, `covered`, `blocked`, `untracked_drift`, or `stale`. An open
sync PR covers drift only when its `sync/workflows-<template-hash>` branch matches the
current compiled plan and it is within the 36-hour coverage lease. Fully covered drift
exits zero and does not append a durable-tracker comment; stale (including expired
coverage), blocked (including global/lookup failures), and untracked states remain
actionable failures.

Comment thread
coderabbitai[bot] marked this conversation as resolved.
### Adding a New Consumer Repo

1. Add the repo to `REGISTERED_CONSUMER_REPOS` in `maint-68-sync-consumer-repos.yml`.
Expand Down
158 changes: 153 additions & 5 deletions scripts/check_consumer_sync_drift.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import hashlib
import json
import os
from datetime import UTC, datetime, timedelta
from pathlib import Path

import requests
Expand Down Expand Up @@ -35,6 +36,7 @@
SUMMARY_ITEM_LIMIT = 50
CONTENT_ERROR_THRESHOLD = 5
SYNC_BRANCH_PREFIX = "sync/workflows-"
SYNC_COVERAGE_MAX_AGE = timedelta(hours=36)
TOKEN_ENV_ORDER = (
"DRIFT_TOKEN",
"SERVICE_BOT_PAT",
Expand Down Expand Up @@ -371,6 +373,7 @@ def fetch_open_sync_prs(
branch = str(head.get("ref", "")).strip()
if not branch.startswith(SYNC_BRANCH_PREFIX):
continue
head_repo = head.get("repo") if isinstance(head.get("repo"), dict) else {}
prs.append(
{
"repo": repo,
Expand All @@ -379,6 +382,7 @@ def fetch_open_sync_prs(
"url": item.get("html_url", ""),
"branch": branch,
"head_sha": head.get("sha", ""),
"head_repo": head_repo.get("full_name", ""),
"created_at": item.get("created_at", ""),
"updated_at": item.get("updated_at", ""),
}
Expand All @@ -394,6 +398,112 @@ def fetch_open_sync_prs(
return prs, None


def parse_github_timestamp(value: object) -> datetime | None:
"""Parse GitHub's UTC timestamp shape without making malformed data current."""
if not isinstance(value, str) or not value:
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
Comment thread
Copilot marked this conversation as resolved.
if parsed.tzinfo is None:
return None
return parsed.astimezone(UTC)


def build_remediation_states(
*,
repos: list[str],
drift: set[str],
missing: set[str],
errors: set[str],
obsolete: set[str],
open_sync_prs: list[dict[str, object]],
sync_pr_lookup_errors: list[str],
expected_branch: str,
global_errors: list[str] | None = None,
now: datetime | None = None,
) -> dict[str, dict[str, object]]:
"""Classify drift using the current compiler-plan branch, not PR presence alone."""
current_time = now or datetime.now(UTC)
gaps_by_repo: dict[str, list[str]] = {repo: [] for repo in repos}
for category, items in (
("drift", drift),
("missing", missing),
("errors", errors),
("obsolete", obsolete),
):
for item in items:
repo, _detail = split_report_item(item)
if repo in gaps_by_repo:
gaps_by_repo[repo].append(category)
Comment thread
stranske marked this conversation as resolved.

global_errors = global_errors or []
lookup_failed = {item.split(":", 1)[0] for item in sync_pr_lookup_errors}
prs_by_repo: dict[str, list[dict[str, object]]] = {repo: [] for repo in repos}
for pr in open_sync_prs:
repo = str(pr.get("repo", ""))
if repo in prs_by_repo:
prs_by_repo[repo].append(pr)

states: dict[str, dict[str, object]] = {}
for repo in repos:
categories = sorted(set(gaps_by_repo[repo]))
if global_errors:
states[repo] = {
"state": "blocked",
"reason": "global comparison error",
"categories": categories,
"global_errors": global_errors,
}
continue
if not categories:
states[repo] = {"state": "converged", "reason": "no drift detected"}
continue
if repo in lookup_failed or "errors" in categories:
states[repo] = {
"state": "blocked",
"reason": "lookup or content error",
"categories": categories,
Comment thread
stranske marked this conversation as resolved.
}
continue

current = [
pr
for pr in prs_by_repo[repo]
if pr.get("branch") == expected_branch and pr.get("head_repo") == repo
]
fresh = []
for pr in current:
updated = parse_github_timestamp(pr.get("updated_at"))
if updated is not None and current_time - updated <= SYNC_COVERAGE_MAX_AGE:
fresh.append(pr)
if fresh:
states[repo] = {
"state": "covered",
"reason": "current compiler-plan sync PR is open",
"expected_branch": expected_branch,
"pr": fresh[0],
"categories": categories,
}
elif current:
states[repo] = {
"state": "stale",
"reason": "current compiler-plan sync PR exceeded coverage lease",
"expected_branch": expected_branch,
"pr": current[0],
"categories": categories,
}
else:
states[repo] = {
"state": "untracked_drift",
"reason": "no open sync PR matches the current compiler plan",
"expected_branch": expected_branch,
"categories": categories,
}
return states


def record_content_error(
*,
errors: set[str],
Expand Down Expand Up @@ -428,6 +538,8 @@ def build_report(
open_sync_prs: list[dict[str, object]] | None = None,
sync_pr_lookup_errors: list[str] | None = None,
token_diagnostics: dict[str, object] | None = None,
current_plan_id: str = "",
now: datetime | None = None,
) -> dict[str, object]:
skipped = skipped or set()
open_sync_prs = open_sync_prs or []
Expand All @@ -438,7 +550,38 @@ def build_report(
"errors": len(errors),
"obsolete": len(obsolete),
}
status = "pass" if all(value == 0 for value in counts.values()) else "drift"
expected_branch = ""
if current_plan_id.startswith("sha256:"):
expected_branch = f"{SYNC_BRANCH_PREFIX}{current_plan_id.split(':', 1)[1][:12]}"
known_repos = set(repos)
global_errors = sorted(
item
for item in [*errors, *sync_pr_lookup_errors]
if split_report_item(item)[0] not in known_repos
)
if not repos:
global_errors.append("no registered consumer repositories supplied")
remediation_states = build_remediation_states(
repos=repos,
drift=drift,
missing=missing,
errors=errors,
obsolete=obsolete,
open_sync_prs=open_sync_prs,
sync_pr_lookup_errors=sync_pr_lookup_errors,
expected_branch=expected_branch,
global_errors=global_errors,
now=now,
)
state_values = {str(item["state"]) for item in remediation_states.values()}
if global_errors:
status = "drift"
elif state_values <= {"converged"}:
status = "converged"
elif state_values <= {"converged", "covered"}:
status = "covered"
else:
status = "drift"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
repo_summaries = build_repo_summaries(
repos=repos,
drift=drift,
Expand All @@ -450,9 +593,7 @@ def build_report(
targeted_repos = [str(item["repo"]) for item in top_repo_gaps]
open_sync_repo_count = len({str(item.get("repo", "")) for item in open_sync_prs if item})
latest_open_sync_pr = open_sync_prs[0] if open_sync_prs else None
remediation_state = "pass"
if status != "pass":
remediation_state = "pending_sync_prs" if open_sync_prs else "needs_sync"
remediation_state = status
report: dict[str, object] = {
"schema": REPORT_SCHEMA,
"status": status,
Expand Down Expand Up @@ -487,6 +628,11 @@ def build_report(
},
"sync_remediation": {
"state": remediation_state,
"plan_id": current_plan_id,
"expected_branch": expected_branch,
"coverage_lease_hours": int(SYNC_COVERAGE_MAX_AGE.total_seconds() // 3600),
"repo_states": remediation_states,
"global_errors": global_errors,
"open_pr_count": len(open_sync_prs),
"repo_count": open_sync_repo_count,
"latest_open_pr": latest_open_sync_pr,
Expand Down Expand Up @@ -738,6 +884,7 @@ def main() -> int:
return 1

sections = list(COPY_SYNCED_SECTIONS)
current_plan_id = str(compiled.to_plan()["plan_id"])

session, token_diagnostics = select_read_token(
candidates=candidates,
Expand Down Expand Up @@ -848,11 +995,12 @@ def _check_file(local_file: Path, remote_target: str, repo: str) -> None:
open_sync_prs=open_sync_prs,
sync_pr_lookup_errors=sync_pr_lookup_errors,
token_diagnostics=token_diagnostics,
current_plan_id=current_plan_id,
)
write_report_json(args.report_json, report)
write_summary_markdown(args.summary, report)

if report["status"] != "pass":
if report["status"] not in {"converged", "covered"}:
print("::warning::Consumer repo drift detected")
return 1

Expand Down
Loading
Loading