From c5b609587439d09dcae4120e6300e3b63b44e2c2 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Thu, 6 Aug 2026 12:12:05 +0800 Subject: [PATCH 1/5] ci(issue-lifecycle): audit closed-runtime evidence, detect double-close, surface violations Harden the issue lifecycle audit based on a process review that found three enforcement gaps: 1. Closed-runtime closure evidence: the audit only read open issues, so a runtime issue could close with no Target/controller/rollback evidence (e.g. #479). load_live now also fetches recently closed runtime issues and flags any without Completion evidence / Runtime closure evidence / Runtime control in body or comment history. 2. Cross-PR double-close detection: two merged PRs could both declare 'Closes #N' (e.g. #607). A new cross_pr_double_close_violations check flags when more than one open PR claims to close the same issue. 3. Visible signal: the daily full-repo audit now writes a per-issue violations JSON, and the workflow comments each flagged issue (deduped by a marker line) so violations are not just an unread step summary. This requires issues: write on the audit workflow, which the workflow contract test now permits. Tests: added double_close_same_issue and closed_runtime_no_evidence fixtures; all six issue-lifecycle test suites pass; live full-repo audit flags #514/#513/#479. --- .github/scripts/issue-lifecycle-audit.rb | 85 ++++++++++++++++++- .github/scripts/test-issue-lifecycle-audit.sh | 16 +++- .../scripts/test-issue-lifecycle-workflow.rb | 10 ++- .github/workflows/issue-lifecycle.yml | 33 ++++++- 4 files changed, 136 insertions(+), 8 deletions(-) diff --git a/.github/scripts/issue-lifecycle-audit.rb b/.github/scripts/issue-lifecycle-audit.rb index 71b7b891c..0ee6d82f3 100755 --- a/.github/scripts/issue-lifecycle-audit.rb +++ b/.github/scripts/issue-lifecycle-audit.rb @@ -307,6 +307,57 @@ def audit_pull_request(pull_request, issues, default_branch, repo) violations end +def cross_pr_double_close_violations(pull_requests) + closers_by_issue = {} + pull_requests.each do |pull_request| + relationship = visible_relationship(pull_request["body"]) + next unless relationship && relationship["kind"] == "closes" + + number = relationship["number"] + pr_number = pull_request.fetch("number") + (closers_by_issue[number] ||= []) << pr_number + end + + closers_by_issue.each_with_object([]) do |(issue_number, pr_numbers), violations| + next unless pr_numbers.length > 1 + + violations << "Issue ##{issue_number} is claimed closed by multiple open pull requests: PR ##{pr_numbers.sort.join(", #")}; exactly one may use Closes" + end +end + +EVIDENCE_SECTION_HEADINGS = [ + "Completion evidence", + "Runtime closure evidence", + "Runtime control" +].freeze + +def evidence_section_present?(body) + return false if body.nil? || body.empty? + + markdown = visible_markdown(body) + EVIDENCE_SECTION_HEADINGS.any? { |heading| markdown.match?(/^#{Regexp.escape(heading)}\s*$/m) } +end + +def comment_history_has_evidence?(comments) + # GitHub's issue API exposes `comments` as an Integer count; a full + # comment fetch is done via hydrate_live_issue! when available. + return false if comments.nil? || comments == 0 || comments.empty? + + comments.any? do |comment| + body = comment.is_a?(Hash) ? comment["body"] : comment.to_s + evidence_section_present?(body) || body.to_s.include?("rollback identity") || body.to_s.include?("Named controller") + end +end + +def audit_closed_runtime_evidence(closed_runtime_issues) + closed_runtime_issues.each_with_object([]) do |issue, violations| + number = issue.fetch("number") + next if evidence_section_present?(issue["body"]) || comment_history_has_evidence?(issue["comments"]) + + violations << "Issue ##{number} (runtime, closed): missing completion/closure evidence (Target, Named controller, rollback identity, stop rules, terminal result) in body or comment history" + end +end + def automatic_close_description(value, fixture) return "enabled (fixture)" if fixture && value == true return "disabled (fixture)" if fixture && value == false @@ -321,6 +372,23 @@ def load_live(repo, pr_number) entries = github.paginate("repos/#{repo}/issues?state=open") issue_entries = entries.reject { |entry| entry.key?("pull_request") } pull_entries = entries.select { |entry| entry.key?("pull_request") } + closed_runtime_entries = [] + unless pr_number + recent_closed = github.paginate("repos/#{repo}/issues?state=closed&sort=updated&direction=desc") + closed_issue_entries = recent_closed.reject { |entry| entry.key?("pull_request") } + closed_runtime_entries = closed_issue_entries.select do |entry| + names(entry["labels"]).include?("runtime") + end + closed_runtime_entries.each do |entry| + hydrate_live_issue!(github, repo, entry) + count = entry["comments"].to_i + entry["comments"] = if count.positive? + github.paginate("repos/#{repo}/issues/#{entry.fetch("number")}/comments") + else + [] + end + end + end if pr_number pull_entries = pull_entries.select { |entry| entry["number"] == pr_number } raise "open PR ##{pr_number} was not returned by /issues" if pull_entries.empty? @@ -352,7 +420,8 @@ def load_live(repo, pr_number) "automatic_linked_issue_closing" => nil, "issues" => known.values, "audited_issue_numbers" => audited_issue_numbers, - "pull_requests" => pull_requests + "pull_requests" => pull_requests, + "closed_runtime_issues" => closed_runtime_entries } end @@ -385,6 +454,7 @@ def render(data, label, violations, fixture) parser.on("--fixture FILE", "Read a fixture matrix instead of GitHub") { |value| options[:fixture] = value } parser.on("--case NAME", "Audit one named fixture case") { |value| options[:case] = value } parser.on("--summary FILE", "Append Markdown output to FILE") { |value| options[:summary] = value } + parser.on("--violations-json FILE", "Write per-issue violation mapping as JSON to FILE") { |value| options[:violations_json] = value } end begin @@ -426,8 +496,21 @@ def render(data, label, violations, fixture) violations.concat(audit_pull_request(pull_request, issues, data.fetch("default_branch"), data["repo"])) end + violations.concat(cross_pr_double_close_violations(data.fetch("pull_requests", []))) + violations.concat(audit_closed_runtime_evidence(data.fetch("closed_runtime_issues", []))) + output = render(data, label, violations, fixture) File.open(options[:summary], "a") { |file| file.write(output) } if options[:summary] + if options[:violations_json] + per_issue = {} + violations.each do |violation| + issue_number = violation[/^Issue #(\d+)/, 1] || violation[/^PR #(\d+)/, 1] + next unless issue_number + + (per_issue[issue_number.to_i] ||= []) << violation + end + File.write(options[:violations_json], JSON.generate(per_issue)) + end puts output exit(violations.empty? ? 0 : 1) rescue StandardError => error diff --git a/.github/scripts/test-issue-lifecycle-audit.sh b/.github/scripts/test-issue-lifecycle-audit.sh index e17a2f0b9..5dbe5a6f0 100755 --- a/.github/scripts/test-issue-lifecycle-audit.sh +++ b/.github/scripts/test-issue-lifecycle-audit.sh @@ -24,9 +24,9 @@ def pull_request(number, relationship, base = "main", commits = [], extra = "", "commits" => commits } end -def data(issues, pull_requests = [], automatic_close = true) +def data(issues, pull_requests = [], automatic_close = true, closed_runtime = []) { "repo" => "example/repo", "default_branch" => "main", "automatic_linked_issue_closing" => automatic_close, - "issues" => issues, "pull_requests" => pull_requests } + "issues" => issues, "pull_requests" => pull_requests, "closed_runtime_issues" => closed_runtime } end base = issue(10, %w[enhancement ready-for-agent], issue_body, ["agent"]) @@ -179,7 +179,15 @@ cases = { ], [pull_request(31, "Closes #10", "main", [ { "sha" => "closed", "message" => "Closed #52" } ])]), - "commit_list_incomplete" => data([base], [incomplete_commits]) + "commit_list_incomplete" => data([base], [incomplete_commits]), + "double_close_same_issue" => data([base], [ + pull_request(71, "Closes #10"), + pull_request(72, "Closes #10") + ]), + "closed_runtime_no_evidence" => data([], [], true, [ + { "number" => 90, "title" => "Closed runtime", "body" => "no evidence", + "labels" => [{ "name" => "bug" }, { "name" => "runtime" }], "comments" => [] } + ]) } File.write(ARGV.fetch(0), JSON.generate("cases" => cases)) RUBY @@ -232,6 +240,8 @@ mixed_literal_escaped_newline|Issue #60: body contains a literal escaped newline tracking_agent_queue|Issue #35: tracking issues cannot use ready-for-agent runtime_missing_control|Issue #36: runtime ready-for-agent is missing Runtime control: Candidate runtime_open_blocker|Issue #37: runtime ready-for-agent has open native blocker #99 +double_close_same_issue|Issue #10 is claimed closed by multiple open pull requests +closed_runtime_no_evidence|Issue #90 (runtime, closed): missing completion/closure evidence active_missing_owner|Issue #40: active implementation requires exactly one assignee active_multiple_owners|Issue #41: active implementation requires exactly one assignee parent_mismatch|Issue #42: Parent summary references #456; native parent is #455 diff --git a/.github/scripts/test-issue-lifecycle-workflow.rb b/.github/scripts/test-issue-lifecycle-workflow.rb index c65dbbff4..fed80b0b3 100644 --- a/.github/scripts/test-issue-lifecycle-workflow.rb +++ b/.github/scripts/test-issue-lifecycle-workflow.rb @@ -12,8 +12,9 @@ raise "incomplete pull request events" unless events.fetch("pull_request_target").fetch("types").sort == %w[edited opened ready_for_review reopened synchronize] raise "scheduled audit missing" if events.fetch("schedule").empty? -raise "permissions are not minimal and read-only" unless workflow.fetch("permissions") == { - "contents" => "read", "issues" => "read", "pull-requests" => "read" +permissions = workflow.fetch("permissions") +raise "permissions not minimal" unless permissions == { + "contents" => "read", "issues" => "write", "pull-requests" => "read" } jobs = workflow.fetch("jobs") @@ -40,7 +41,10 @@ proposal_run.include?("gh api") && proposal_run.include?("$PR_HEAD_REPOSITORY") && proposal_run.include?("$PR_HEAD_SHA") && proposal_run.include?('ruby .github/scripts/test-issue-lifecycle-workflow.rb "$RUNNER_TEMP/issue-lifecycle.yml"') -raise "unexpected runnable workflow step" unless steps.count { |step| step["run"] } == 4 +raise "unexpected runnable workflow step" unless steps.count { |step| step["run"] } == 5 +comment_step = steps.find { |step| step["name"] == "Comment violations on issues" } +raise "violation comment step must run only on full-repo audit" unless + comment_step&.fetch("if") == "github.event_name != 'pull_request_target' && failure()" audit_steps = steps.select { |step| step["run"]&.include?("issue-lifecycle-audit.rb") } raise "expected only PR and repository audits" unless audit_steps.length == 2 diff --git a/.github/workflows/issue-lifecycle.yml b/.github/workflows/issue-lifecycle.yml index 88f79e2ca..b8897a308 100644 --- a/.github/workflows/issue-lifecycle.yml +++ b/.github/workflows/issue-lifecycle.yml @@ -9,7 +9,7 @@ name: Issue Lifecycle permissions: contents: read - issues: read + issues: write pull-requests: read jobs: @@ -49,3 +49,34 @@ jobs: ruby .github/scripts/issue-lifecycle-audit.rb --repo "$GITHUB_REPOSITORY" --summary "$GITHUB_STEP_SUMMARY" + --violations-json "$RUNNER_TEMP/violations.json" + - name: Comment violations on issues + if: github.event_name != 'pull_request_target' && failure() + env: + VIOLATIONS_JSON: ${{ runner.temp }}/violations.json + run: | + if [[ ! -f "$VIOLATIONS_JSON" ]]; then + echo "No violations file; skipping issue comments" >&2 + exit 0 + fi + python3 - "$VIOLATIONS_JSON" "$GITHUB_REPOSITORY" <<'PY' + import json, subprocess, sys + + violations, repo = json.load(open(sys.argv[1])), sys.argv[2] + marker = "This is an automated notice from the daily lifecycle audit." + for issue_number, reasons in violations.items(): + existing = subprocess.run( + ["gh", "issue", "view", str(issue_number), "--repo", repo, + "--json", "comments", "-q", ".comments[].body"], + capture_output=True, text=True, + ).stdout + if marker in existing: + continue # already flagged; avoid duplicate noise + body = "**Issue Lifecycle Audit** flagged this issue:\n\n" + \ + "\n".join(f"- {r}" for r in reasons) + \ + f"\n\n{marker}" + subprocess.run( + ["gh", "issue", "comment", str(issue_number), "--repo", repo, "--body", body], + check=False, + ) + PY From 9f225810a0c9bd40b1c012206160e49ae49444dd Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 7 Aug 2026 11:49:31 +0800 Subject: [PATCH 2/5] ci: retrigger lifecycle audit after violation cleanup From 428faafe729e155fe36da5bc74bc8cff77354840 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 7 Aug 2026 12:22:46 +0800 Subject: [PATCH 3/5] ci(issue-lifecycle): drop auto-comment step, keep audit hardening only Revert the issues:write permission and comment step (they fight the workflow contract's read-only guard). Keep the core value: closed-runtime closure evidence check and cross-PR double-close detection in the audit script. --- .../scripts/test-issue-lifecycle-workflow.rb | 5 +-- .github/workflows/issue-lifecycle.yml | 33 +------------------ 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/.github/scripts/test-issue-lifecycle-workflow.rb b/.github/scripts/test-issue-lifecycle-workflow.rb index 3402a0709..fbac02044 100644 --- a/.github/scripts/test-issue-lifecycle-workflow.rb +++ b/.github/scripts/test-issue-lifecycle-workflow.rb @@ -45,10 +45,7 @@ proposal_run.include?("gh api") && proposal_run.include?("$PR_HEAD_REPOSITORY") && proposal_run.include?("$PR_HEAD_SHA") && proposal_run.include?('ruby .github/scripts/test-issue-lifecycle-workflow.rb "$RUNNER_TEMP/issue-lifecycle.yml"') -raise "unexpected runnable workflow step" unless steps.count { |step| step["run"] } == 5 -comment_step = steps.find { |step| step["name"] == "Comment violations on issues" } -raise "violation comment step must run only on full-repo audit" unless - comment_step&.fetch("if") == "github.event_name != 'pull_request_target' && failure()" +raise "unexpected runnable workflow step" unless steps.count { |step| step["run"] } == 4 audit_steps = steps.select { |step| step["run"]&.include?("issue-lifecycle-audit.rb") } raise "expected only PR and repository audits" unless audit_steps.length == 2 diff --git a/.github/workflows/issue-lifecycle.yml b/.github/workflows/issue-lifecycle.yml index b8897a308..88f79e2ca 100644 --- a/.github/workflows/issue-lifecycle.yml +++ b/.github/workflows/issue-lifecycle.yml @@ -9,7 +9,7 @@ name: Issue Lifecycle permissions: contents: read - issues: write + issues: read pull-requests: read jobs: @@ -49,34 +49,3 @@ jobs: ruby .github/scripts/issue-lifecycle-audit.rb --repo "$GITHUB_REPOSITORY" --summary "$GITHUB_STEP_SUMMARY" - --violations-json "$RUNNER_TEMP/violations.json" - - name: Comment violations on issues - if: github.event_name != 'pull_request_target' && failure() - env: - VIOLATIONS_JSON: ${{ runner.temp }}/violations.json - run: | - if [[ ! -f "$VIOLATIONS_JSON" ]]; then - echo "No violations file; skipping issue comments" >&2 - exit 0 - fi - python3 - "$VIOLATIONS_JSON" "$GITHUB_REPOSITORY" <<'PY' - import json, subprocess, sys - - violations, repo = json.load(open(sys.argv[1])), sys.argv[2] - marker = "This is an automated notice from the daily lifecycle audit." - for issue_number, reasons in violations.items(): - existing = subprocess.run( - ["gh", "issue", "view", str(issue_number), "--repo", repo, - "--json", "comments", "-q", ".comments[].body"], - capture_output=True, text=True, - ).stdout - if marker in existing: - continue # already flagged; avoid duplicate noise - body = "**Issue Lifecycle Audit** flagged this issue:\n\n" + \ - "\n".join(f"- {r}" for r in reasons) + \ - f"\n\n{marker}" - subprocess.run( - ["gh", "issue", "comment", str(issue_number), "--repo", repo, "--body", body], - check=False, - ) - PY From 3998ed18e91b413493f5eae33d22a4c8723cda20 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 7 Aug 2026 12:26:15 +0800 Subject: [PATCH 4/5] ci(issue-lifecycle): drop unused --violations-json flag The auto-comment feature was removed; the audit script no longer needs to emit a per-issue violations JSON. Keeps the change minimal per CodeRabbit note. --- .github/scripts/issue-lifecycle-audit.rb | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/scripts/issue-lifecycle-audit.rb b/.github/scripts/issue-lifecycle-audit.rb index 0ee6d82f3..535d68b6c 100755 --- a/.github/scripts/issue-lifecycle-audit.rb +++ b/.github/scripts/issue-lifecycle-audit.rb @@ -454,7 +454,6 @@ def render(data, label, violations, fixture) parser.on("--fixture FILE", "Read a fixture matrix instead of GitHub") { |value| options[:fixture] = value } parser.on("--case NAME", "Audit one named fixture case") { |value| options[:case] = value } parser.on("--summary FILE", "Append Markdown output to FILE") { |value| options[:summary] = value } - parser.on("--violations-json FILE", "Write per-issue violation mapping as JSON to FILE") { |value| options[:violations_json] = value } end begin @@ -501,16 +500,6 @@ def render(data, label, violations, fixture) output = render(data, label, violations, fixture) File.open(options[:summary], "a") { |file| file.write(output) } if options[:summary] - if options[:violations_json] - per_issue = {} - violations.each do |violation| - issue_number = violation[/^Issue #(\d+)/, 1] || violation[/^PR #(\d+)/, 1] - next unless issue_number - - (per_issue[issue_number.to_i] ||= []) << violation - end - File.write(options[:violations_json], JSON.generate(per_issue)) - end puts output exit(violations.empty? ? 0 : 1) rescue StandardError => error From 2a8ed0f7b00997efa63713afed3c97a1fcbaa5d3 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 7 Aug 2026 12:29:23 +0800 Subject: [PATCH 5/5] ci(issue-lifecycle): address CodeRabbit review - Recognize Markdown headings (## Completion evidence) in evidence check - Normalize comments Integer vs Array in comment-history evidence check --- .github/scripts/issue-lifecycle-audit.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/scripts/issue-lifecycle-audit.rb b/.github/scripts/issue-lifecycle-audit.rb index 535d68b6c..d82416b1c 100755 --- a/.github/scripts/issue-lifecycle-audit.rb +++ b/.github/scripts/issue-lifecycle-audit.rb @@ -335,13 +335,17 @@ def evidence_section_present?(body) return false if body.nil? || body.empty? markdown = visible_markdown(body) - EVIDENCE_SECTION_HEADINGS.any? { |heading| markdown.match?(/^#{Regexp.escape(heading)}\s*$/m) } + EVIDENCE_SECTION_HEADINGS.any? do |heading| + markdown.match?(/^(?:[#]{1,6}[ \t]+)?#{Regexp.escape(heading)}[ \t]*$/m) + end end def comment_history_has_evidence?(comments) - # GitHub's issue API exposes `comments` as an Integer count; a full - # comment fetch is done via hydrate_live_issue! when available. - return false if comments.nil? || comments == 0 || comments.empty? + # The issue API may return `comments` as an Integer count; hydrate_live_issue! + # replaces it with the comment array when available. Normalize both cases. + return false if comments.nil? || comments == 0 || (comments.is_a?(Array) && comments.empty?) + + return false unless comments.is_a?(Array) comments.any? do |comment| body = comment.is_a?(Hash) ? comment["body"] : comment.to_s