diff --git a/.github/scripts/issue-lifecycle-audit.rb b/.github/scripts/issue-lifecycle-audit.rb new file mode 100755 index 000000000..71b7b891c --- /dev/null +++ b/.github/scripts/issue-lifecycle-audit.rb @@ -0,0 +1,438 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "open3" +require "optparse" + +CATEGORIES = %w[bug enhancement].freeze +TRIAGE_STATES = %w[needs-triage needs-info ready-for-agent ready-for-human wontfix].freeze +RUNTIME_CONTROL_FIELDS = { + "Target" => /\A(?:[-*]\s*)?(?:\*\*)?(?:Target|Exact target identity)(?:\*\*)?\s*:\s*(.+)\z/i, + "Candidate" => /\A(?:[-*]\s*)?(?:\*\*)?(?:Candidate|Candidate and configuration identity)(?:\*\*)?\s*:\s*(.+)\z/i, + "Controller" => /\A(?:[-*]\s*)?(?:\*\*)?(?:Controller|Named controller)(?:\*\*)?\s*:\s*(.+)\z/i, + "Stop rule" => /\A(?:[-*]\s*)?(?:\*\*)?Stop rules?(?:\*\*)?\s*:\s*(.+)\z/i, + "Rollback" => /\A(?:[-*]\s*)?(?:\*\*)?(?:Rollback|Rollback identity)(?:\*\*)?\s*:\s*(.+)\z/i +}.freeze +CLOSING_KEYWORD_SOURCE = "(?:close[sd]?|fix(?:es|ed)?|resolve[sd]?)" +QUALIFIED_ISSUE_REFERENCE = "(?:([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+))?#(\\d+)\\b" +CLOSING_PATTERN = Regexp.new("\\b(#{CLOSING_KEYWORD_SOURCE})\\s*:?\\s+#{QUALIFIED_ISSUE_REFERENCE}", Regexp::IGNORECASE) +NEGATED_CLOSING_PATTERN = Regexp.new( + "\\b(?:(?:do(?:es)?|did|will|would|should|can|could|must)\\s+not|cannot|doesn't|don't|didn't|won't|can't|never|not)(?:\\s+[A-Za-z-]+){0,3}\\s+#{CLOSING_KEYWORD_SOURCE}\\s*:?\\s+#{QUALIFIED_ISSUE_REFERENCE}", + Regexp::IGNORECASE +) + +class GitHubReadOnly + API_VERSION = "2026-03-10" + + def initialize(repo) + raise "invalid repository #{repo.inspect}; expected OWNER/REPO" unless repo.to_s.match?(/\A[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\z/) + + @repo = repo + end + + attr_reader :repo + + def get(path, allow_404 = false) + stdout, stderr, status = Open3.capture3( + "gh", "api", "--method", "GET", + "-H", "Accept: application/vnd.github+json", + "-H", "X-GitHub-Api-Version: #{API_VERSION}", + path + ) + return nil if allow_404 && !status.success? && stderr.include?("HTTP 404") + raise "GitHub API GET #{path} failed: #{stderr.strip}" unless status.success? + + JSON.parse(stdout) + end + + def paginate(path) + page = 1 + items = [] + loop do + separator = path.include?("?") ? "&" : "?" + batch = get("#{path}#{separator}per_page=100&page=#{page}") + raise "GitHub API GET #{path} did not return an array" unless batch.is_a?(Array) + + items.concat(batch) + break if batch.length < 100 + + page += 1 + end + items + end +end +def markdown_section(markdown, title) + match = markdown.to_s.match(/^##(?:#)?[ \t]+#{Regexp.escape(title)}[ \t]*\r?\n(?.*?)(?=^##(?:#)?[ \t]+|\z)/mi) + match && match[:body] +end + +def visible_markdown(body) + body.to_s.gsub(//m, "") +end + +def names(values) + Array(values).map { |value| value.is_a?(Hash) ? value["name"] || value["login"] : value }.compact +end + +def native_parent_number(issue) + parent = issue["parent"] + parent.is_a?(Hash) ? parent["number"] : parent +end + +def blockers(issue) + Array(issue["blocked_by"]).map do |blocker| + blocker.is_a?(Hash) ? blocker : { "number" => blocker, "state" => "open" } + end +end + +def summary_numbers(body, title) + section = markdown_section(visible_markdown(body), title) + section && section.scan(/#(\d+)/).flatten.map(&:to_i).uniq.sort +end + +def references(numbers) + Array(numbers).empty? ? "None" : Array(numbers).map { |number| "##{number}" }.join(", ") +end + +def invalid_control_value?(value) + value.to_s.strip.empty? || value.to_s.strip.match?(/\A(?:none|n\/a|tbd|unknown|-)\z/i) +end + +def issue_form_value(body, *titles) + titles.each do |title| + section = markdown_section(visible_markdown(body), title) + next unless section + + value = section.lines.map(&:strip).reject(&:empty?).join(" ") + return value unless value.empty? + end + + nil +end + +def hydrate_live_issue!(github, repo, issue) + return unless issue && !issue.key?("pull_request") + + parent_url = issue["parent_issue_url"].to_s + issue["parent"] = parent_url[/\/issues\/(\d+)\z/, 1]&.to_i + summary = issue["issue_dependencies_summary"] + issue["blocked_by"] = if summary && summary["total_blocked_by"].to_i.zero? + [] + else + github.paginate("repos/#{repo}/issues/#{issue.fetch("number")}/dependencies/blocked_by") + end +end + +def missing_runtime_control(body) + section = markdown_section(visible_markdown(body), "Runtime control") + unless section + field_values = { + "Target" => issue_form_value(body, "Exact target identity", "Target"), + "Candidate" => issue_form_value(body, "Candidate and configuration identity", "Candidate"), + "Controller" => issue_form_value(body, "Named controller", "Controller"), + "Stop rule" => issue_form_value(body, "Stop rules", "Stop rule"), + "Rollback" => issue_form_value(body, "Rollback identity and procedure", "Rollback identity", "Rollback") + } + return field_values.each_with_object([]) { |(name, value), missing| missing << name if invalid_control_value?(value) } + end + + lines = section.lines.map(&:strip).reject(&:empty?) + RUNTIME_CONTROL_FIELDS.each_with_object([]) do |(name, pattern), missing| + value = lines.map { |line| line.match(pattern) }.compact.map { |match| match[1].strip }.first + missing << name if invalid_control_value?(value) + end +end + +def literal_escaped_newline_artifact?(body) + text = body.to_s + return false unless text.include?("\\n") + + !text.include?("\n") || text.match?(/(?:\A|\n)[#]{1,6}[ \t]+[^\\\r\n]+\\n(?:\\n)?/) +end + +def active_owner_violation(issue) + assignee_count = names(issue["assignees"]).length + return if assignee_count == 1 + + "Issue ##{issue.fetch("number")}: active implementation requires exactly one assignee; found #{assignee_count}" +end + +def audit_issue(issue, has_open_pr) + number = issue.fetch("number") + labels = names(issue["labels"]) + violations = [] + categories = labels & CATEGORIES + states = labels & TRIAGE_STATES + + violations << "Issue ##{number}: expected exactly one category label; found #{categories.empty? ? "none" : categories.join(", ")}" unless categories.length == 1 + violations << "Issue ##{number}: expected exactly one triage state label; found #{states.empty? ? "none" : states.join(", ")}" unless states.length == 1 + violations << "Issue ##{number}: body contains a literal escaped newline (\\n); publish multiline Markdown through a body file" if literal_escaped_newline_artifact?(issue["body"]) + violations << "Issue ##{number}: tracking issues cannot use ready-for-agent" if labels.include?("tracking") && labels.include?("ready-for-agent") + + if labels.include?("runtime") && labels.include?("ready-for-agent") + missing = missing_runtime_control(issue["body"]) + violations << "Issue ##{number}: runtime ready-for-agent is missing Runtime control: #{missing.join(", ")}" unless missing.empty? + open_blockers = blockers(issue).reject { |blocker| blocker["state"].to_s.downcase == "closed" }.map { |blocker| blocker["number"] } + violations << "Issue ##{number}: runtime ready-for-agent has open native blocker #{references(open_blockers)}" unless open_blockers.empty? + end + + owner_violation = active_owner_violation(issue) if has_open_pr + violations << owner_violation if owner_violation + + parent_summary = summary_numbers(issue["body"], "Parent") + native_parent = native_parent_number(issue) + native_parents = native_parent ? [native_parent] : [] + if parent_summary && parent_summary != native_parents + violations << "Issue ##{number}: Parent summary references #{references(parent_summary)}; native parent is #{references(native_parents)}" + elsif !parent_summary && !native_parents.empty? + violations << "Issue ##{number}: Parent summary is missing; native parent is #{references(native_parents)}" + end + + blocker_summary = summary_numbers(issue["body"], "Blocked by") + native_blockers = blockers(issue).map { |blocker| blocker["number"] }.compact.uniq.sort + if blocker_summary && blocker_summary != native_blockers + violations << "Issue ##{number}: Blocked by summary references #{references(blocker_summary)}; native blocked-by is #{references(native_blockers)}" + elsif !blocker_summary && !native_blockers.empty? + violations << "Issue ##{number}: Blocked by summary is missing; native blocked-by is #{references(native_blockers)}" + end + + violations +end + +def visible_relationship(body) + section = markdown_section(visible_markdown(body), "Issue relationship") + lines = section.to_s.lines.map(&:strip).reject(&:empty?) + return nil unless lines.length == 1 + return { "kind" => "none" } if lines.first == "None" + + match = lines.first.match(/\A(Refs|Closes) #(\d+)\z/) + match && { "kind" => match[1].downcase, "number" => match[2].to_i } +end + +def relationship_numbers(body) + section = markdown_section(visible_markdown(body), "Issue relationship") + section.to_s.scan(/\b(?:Refs|Closes) #(\d+)\b/).flatten.map(&:to_i).uniq +end + +def closing_keywords(text) + text.to_s.to_enum(:scan, CLOSING_PATTERN).map do + match = Regexp.last_match + { "keyword" => match[1], "repository" => match[2], "number" => match[3].to_i } + end +end + +def closing_description(closing) + target = closing["repository"] ? "#{closing["repository"]}##{closing["number"]}" : "##{closing["number"]}" + "#{closing["keyword"]} #{target}" +end + +def closing_targets_issue?(closing, number, repo) + return false unless closing["number"] == number + + !closing["repository"] || (repo && closing["repository"].casecmp?(repo)) +end + +def relationship_description(relationship) + return "None" if relationship["kind"] == "none" + + "#{relationship["kind"] == "refs" ? "Refs" : "Closes"} ##{relationship["number"]}" +end + +def audit_pull_request(pull_request, issues, default_branch, repo) + number = pull_request.fetch("number") + relationship = visible_relationship(pull_request["body"]) + unless relationship + return ["PR ##{number}: expected exactly one visible issue relationship (Refs #N, Closes #N, or None)"] + end + + violations = [] + visible_body = visible_markdown(pull_request["body"]) + body_closings = closing_keywords(visible_body) + title = pull_request["title"].to_s + title_closings = closing_keywords(title) + violations << "PR ##{number} body: negated closing phrase is forbidden" if visible_body.match?(NEGATED_CLOSING_PATTERN) + violations << "PR ##{number} title: negated closing phrase is forbidden" if title.match?(NEGATED_CLOSING_PATTERN) + + if relationship["kind"] == "closes" + expected_number = relationship["number"] + unexpected = body_closings.reject { |closing| closing_targets_issue?(closing, expected_number, repo) } + if unexpected.any? || body_closings.length != 1 + violations << "PR ##{number} body: visible Closes ##{expected_number} must be the only closing keyword relationship" + end + unexpected_title = title_closings.reject { |closing| closing_targets_issue?(closing, expected_number, repo) } + if unexpected_title.any? || title_closings.length > 1 + violations << "PR ##{number} title: visible Closes ##{expected_number} must be the only closing keyword relationship" + end + else + body_closings.each do |closing| + violations << "PR ##{number} body: closing keyword #{closing_description(closing)} requires visible Closes ##{closing["number"]}" + end + title_closings.each do |closing| + violations << "PR ##{number} title: closing keyword #{closing_description(closing)} requires visible Closes ##{closing["number"]}" + end + end + + target = relationship["number"] && issues[relationship["number"]] + if relationship["number"] && !target + violations << "PR ##{number}: #{relationship_description(relationship)} references an issue unavailable to the audit" + end + + if relationship["kind"] == "closes" + base = pull_request["base"].is_a?(Hash) ? pull_request.dig("base", "ref") : pull_request["base"] + violations << "PR ##{number}: Closes ##{relationship["number"]} targets #{base}, not default branch #{default_branch}" if base != default_branch + target_labels = target ? names(target["labels"]) : [] + violations << "PR ##{number}: runtime issue ##{relationship["number"]} cannot be closed by a pull request" if target_labels.include?("runtime") + violations << "PR ##{number}: tracking issue ##{relationship["number"]} cannot be closed by a pull request" if target_labels.include?("tracking") + end + + commits = Array(pull_request["commits"]) + expected_commit_count = pull_request["expected_commit_count"] + if expected_commit_count && commits.length != expected_commit_count + violations << "PR ##{number}: fetched #{commits.length} of #{expected_commit_count} commit messages; audit cannot prove every commit safe" + end + commits.each_with_index do |commit, index| + message = commit["message"] || commit.dig("commit", "message") || "" + identity = commit["sha"].to_s.empty? ? (index + 1).to_s : commit["sha"].to_s[0, 12] + violations << "PR ##{number} commit #{identity}: negated closing phrase is forbidden" if message.match?(NEGATED_CLOSING_PATTERN) + closing_keywords(message).each do |closing| + if relationship["kind"] != "closes" + violations << "PR ##{number} commit #{identity}: closing keyword #{closing_description(closing)} requires visible Closes ##{closing["number"]}" + elsif !closing_targets_issue?(closing, relationship["number"], repo) + violations << "PR ##{number} commit #{identity}: closing keyword #{closing_description(closing)} conflicts with visible Closes ##{relationship["number"]}" + end + end + end + + violations +end + +def automatic_close_description(value, fixture) + return "enabled (fixture)" if fixture && value == true + return "disabled (fixture)" if fixture && value == false + return "#{value} (fixture)" if fixture && !value.nil? + + "unknown/unavailable via documented API" +end + +def load_live(repo, pr_number) + github = GitHubReadOnly.new(repo) + repository = github.get("repos/#{repo}") + entries = github.paginate("repos/#{repo}/issues?state=open") + issue_entries = entries.reject { |entry| entry.key?("pull_request") } + pull_entries = entries.select { |entry| entry.key?("pull_request") } + if pr_number + pull_entries = pull_entries.select { |entry| entry["number"] == pr_number } + raise "open PR ##{pr_number} was not returned by /issues" if pull_entries.empty? + end + + pull_requests = pull_entries.map do |entry| + pull_request = github.get("repos/#{repo}/pulls/#{entry.fetch("number")}") + pull_request["expected_commit_count"] = pull_request["commits"] + pull_request["commits"] = github.paginate("repos/#{repo}/pulls/#{entry.fetch("number")}/commits") + pull_request + end + + known = issue_entries.each_with_object({}) { |issue, by_number| by_number[issue.fetch("number")] = issue } + linked_issue_numbers = pull_requests.flat_map { |pull_request| relationship_numbers(pull_request["body"]) }.uniq + audited_issue_numbers = pr_number ? linked_issue_numbers : issue_entries.map { |issue| issue.fetch("number") } + linked_issue_numbers.each do |number| + next if known.key?(number) + + issue = github.get("repos/#{repo}/issues/#{number}", true) + known[number] = issue if issue && !issue.key?("pull_request") + end + audited_issue_numbers.each { |number| hydrate_live_issue!(github, repo, known[number]) } + + { + "source" => "live", + "repo" => repo, + "scope" => pr_number ? "pull request ##{pr_number}" : "all open issues and pull requests", + "default_branch" => repository.fetch("default_branch"), + "automatic_linked_issue_closing" => nil, + "issues" => known.values, + "audited_issue_numbers" => audited_issue_numbers, + "pull_requests" => pull_requests + } +end + +def render(data, label, violations, fixture) + status = violations.empty? ? "PASS" : "FAIL" + lines = [ + "#{status} #{label}", + "# Issue Lifecycle Audit", + "", + "- Scope: #{data["scope"] || "fixture case #{label}"}", + "- Repository: #{data["repo"] || "fixture"}", + "- Default branch: `#{data.fetch("default_branch")}`", + "- automatic-linked-issue-closing: #{automatic_close_description(data["automatic_linked_issue_closing"], fixture)}", + "- Result: **#{status}** (#{violations.length} violation#{violations.length == 1 ? "" : "s"})" + ] + if violations.empty? + lines.concat(["", "No lifecycle violations found."]) + else + lines.concat(["", "## Violations", ""]) + violations.each_with_index { |violation, index| lines << "#{index + 1}. #{violation}" } + end + lines.join("\n") + "\n" +end + +options = {} +parser = OptionParser.new do |parser| + parser.banner = "Usage: issue-lifecycle-audit.rb [--repo OWNER/REPO [--pr N] | --fixture FILE --case NAME]" + parser.on("--repo OWNER/REPO", "Audit live GitHub data") { |value| options[:repo] = value } + parser.on("--pr NUMBER", Integer, "Audit only one open pull request") { |value| options[:pr] = value } + parser.on("--fixture FILE", "Read a fixture matrix instead of GitHub") { |value| options[:fixture] = value } + parser.on("--case NAME", "Audit one named fixture case") { |value| options[:case] = value } + parser.on("--summary FILE", "Append Markdown output to FILE") { |value| options[:summary] = value } +end + +begin + parser.parse! + fixture = !!options[:fixture] + if fixture + raise "--fixture requires --case" unless options[:case] + raise "--fixture cannot be combined with --repo or --pr" if options[:repo] || options[:pr] + matrix = JSON.parse(File.read(options[:fixture])) + data = matrix.fetch("cases").fetch(options[:case]) + label = options[:case] + else + repo = options[:repo] || ENV["GITHUB_REPOSITORY"] + raise "--repo OWNER/REPO is required outside GitHub Actions" unless repo + data = load_live(repo, options[:pr]) + label = options[:pr] ? "PR ##{options[:pr]}" : "live" + end + + violations = [] + open_pr_issue_numbers = data.fetch("pull_requests", []).flat_map { |pull_request| relationship_numbers(pull_request["body"]) }.uniq + audited_issue_numbers = data["audited_issue_numbers"] || data.fetch("issues", []).map { |issue| issue.fetch("number") } + data.fetch("issues", []).each do |issue| + next unless audited_issue_numbers.include?(issue.fetch("number")) + + violations.concat(audit_issue(issue, open_pr_issue_numbers.include?(issue.fetch("number")))) + end + data.fetch("issues", []).each do |issue| + number = issue.fetch("number") + next if audited_issue_numbers.include?(number) || !open_pr_issue_numbers.include?(number) + + owner_violation = active_owner_violation(issue) + violations << owner_violation if owner_violation + end + + issues = data.fetch("issues", []).each_with_object({}) do |issue, by_number| + by_number[issue.fetch("number")] = issue + end + data.fetch("pull_requests", []).each do |pull_request| + violations.concat(audit_pull_request(pull_request, issues, data.fetch("default_branch"), data["repo"])) + end + + output = render(data, label, violations, fixture) + File.open(options[:summary], "a") { |file| file.write(output) } if options[:summary] + puts output + exit(violations.empty? ? 0 : 1) +rescue StandardError => error + output = "ERROR issue lifecycle audit\n# Issue Lifecycle Audit\n\n- Result: **ERROR**\n- #{error.message}\n" + warn output + File.open(options[:summary], "a") { |file| file.write(output) } if options[:summary] rescue nil + exit 2 +end diff --git a/.github/scripts/test-issue-lifecycle-audit.sh b/.github/scripts/test-issue-lifecycle-audit.sh new file mode 100755 index 000000000..e17a2f0b9 --- /dev/null +++ b/.github/scripts/test-issue-lifecycle-audit.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +auditor="$repo_root/.github/scripts/issue-lifecycle-audit.rb" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +ruby -rjson - "$tmp_dir/cases.json" <<'RUBY' +def issue_body(parent = "None", blockers = "None", extra = "") + "## Parent\n\n#{parent}\n\n## Blocked by\n\n#{blockers}\n#{extra}" +end + +def issue(number, labels, body = issue_body, assignees = [], parent = nil, blocked_by = []) + { "number" => number, "title" => "Fixture issue #{number}", "body" => body, + "labels" => labels, "assignees" => assignees, "parent" => parent, + "blocked_by" => blocked_by } +end + +def pull_request(number, relationship, base = "main", commits = [], extra = "", title = "Fixture PR #{number}") + { "number" => number, "base" => base, + "title" => title, + "body" => "## Issue relationship\n\n#{relationship}\n\n## Focused validation\n\nFixture proof.\n#{extra}\n", + "commits" => commits } +end + +def data(issues, pull_requests = [], automatic_close = true) + { "repo" => "example/repo", "default_branch" => "main", "automatic_linked_issue_closing" => automatic_close, + "issues" => issues, "pull_requests" => pull_requests } +end + +base = issue(10, %w[enhancement ready-for-agent], issue_body, ["agent"]) +runtime_control = "\n## Runtime control\n\nTarget: repository setting\nCandidate: commit abc with config digest def\nController: release-owner\nStop rule: stop on failed preflight\nRollback: restore the previous setting\n" +runtime_without_candidate = runtime_control.lines.reject { |line| line.start_with?("Candidate:") }.join +issue_form_runtime = <<~MARKDOWN + +### Exact target identity + +repository setting + +### Candidate and configuration identity + +commit abc with config digest def + +### Named controller + +release-owner + +### Rollback identity and procedure + +restore the previous setting + +### Stop rules + +stop on failed preflight +MARKDOWN +incomplete_commits = pull_request(32, "Refs #10", "main", [ + { "sha" => "first", "message" => "Safe fetched commit" } +]) +incomplete_commits["expected_commit_count"] = 251 +cases = { + "valid_refs" => data([base], [pull_request(20, "Refs #10", "feature-stack")]), + "valid_closes" => data([ + issue(11, %w[enhancement ready-for-agent], issue_body, ["agent"]) + ], [pull_request(22, "Closes #11", "main", [{ "sha" => "fix", "message" => "Fixes #11" }])]), + "valid_none" => data([], [pull_request(23, "\nNone")], false), + "valid_needs_info" => data([issue(13, %w[bug needs-info])]), + "valid_wontfix" => data([issue(14, %w[enhancement wontfix])]), + "authorized_runtime_ready" => data([ + issue(12, %w[enhancement ready-for-agent runtime], + issue_body("None", "#99", runtime_control), [], nil, + [{ "number" => 99, "state" => "closed" }]) + ]), + "authorized_runtime_ready_issue_form" => data([ + issue(15, %w[enhancement ready-for-agent runtime], + issue_body("None", "#99", issue_form_runtime), [], nil, + [{ "number" => 99, "state" => "closed" }]) + ]), + "missing_category" => data([issue(30, %w[ready-for-human])]), + "conflicting_category" => data([issue(31, %w[bug enhancement ready-for-human])]), + "missing_state" => data([issue(32, %w[enhancement])]), + "conflicting_state" => data([issue(33, %w[enhancement needs-triage ready-for-human])]), + "literal_escaped_newline" => data([ + issue(34, %w[enhancement ready-for-human], "## Parent\\n\\nNone\\n\\n## Blocked by\\n\\nNone\\n\\n## Details\\n\\nliteral \\n escape") + ]), + "short_literal_escaped_newline" => data([issue(61, %w[enhancement ready-for-human], "Summary\\nDetails")]), + "valid_literal_escaped_newline" => data([ + issue(35, %w[enhancement ready-for-human], issue_body + "\n\n```json\n{\"pattern\":\"line\\\\nnext\"}\n```\n") + ]), + "mixed_literal_escaped_newline" => data([ + issue(60, %w[enhancement ready-for-human], "Normal preface.\n\n## Parent\\n\\nNone\\n\\n## Blocked by\\n\\nNone") + ]), + "tracking_agent_queue" => data([issue(35, %w[enhancement ready-for-agent tracking])]), + "runtime_missing_control" => data([ + issue(36, %w[enhancement ready-for-agent runtime], issue_body("None", "None", runtime_without_candidate)) + ]), + "runtime_open_blocker" => data([ + issue(37, %w[enhancement ready-for-agent runtime], + issue_body("None", "#99", runtime_control), [], nil, + [{ "number" => 99, "state" => "open" }]) + ]), + "active_missing_owner" => data([ + issue(40, %w[enhancement ready-for-agent]) + ], [pull_request(140, "Refs #40")]).merge("audited_issue_numbers" => []), + "active_multiple_owners" => data([ + issue(41, %w[enhancement ready-for-agent], issue_body, %w[first second]) + ], [pull_request(141, "Refs #41")]).merge("audited_issue_numbers" => []), + "parent_mismatch" => data([ + issue(42, %w[enhancement ready-for-human], issue_body("#456"), [], 455) + ]), + "blocker_mismatch" => data([ + issue(43, %w[enhancement ready-for-human], issue_body("None", "#78"), [], nil, + [{ "number" => 77, "state" => "open" }]) + ]), + "invalid_multiple_relationships" => data([], [ + pull_request(21, "Refs #10\nCloses #10\nNone") + ]), + "negated_pr_closing" => data([base], [ + pull_request(24, "Refs #10", "main", [], "This does not close #10.") + ]), + "non_default_closes" => data([base], [pull_request(25, "Closes #10", "stack")]), + "runtime_closes" => data([ + issue(50, %w[enhancement ready-for-human runtime], issue_body, ["operator"]) + ], [pull_request(26, "Closes #50")]), + "tracking_closes" => data([ + issue(51, %w[enhancement ready-for-human tracking], issue_body, ["maintainer"]) + ], [pull_request(27, "Closes #51")]), + "pr_fix_with_refs" => data([base], [ + pull_request(28, "Refs #10", "main", [], "Fixes #10") + ]), + "qualified_pr_fix_with_refs" => data([base], [ + pull_request(36, "Refs #10", "main", [], "Fixes proerror77/monday#10") + ]), + "qualified_negated_pr_closing" => data([base], [ + pull_request(39, "Refs #10", "main", [], "This does not close proerror77/monday#10.") + ]), + "modified_negated_pr_closing" => data([base], [ + pull_request(43, "Closes #10", "main", [], "", "This does not fully close #10.") + ]), + "pr_title_fix_with_refs" => data([base], [ + pull_request(34, "Refs #10", "main", [], "", "Fixes #10 in title") + ]), + "qualified_pr_title_fix_with_refs" => data([base], [ + pull_request(37, "Refs #10", "main", [], "", "Fixes proerror77/monday#10 in title") + ]), + "commit_fix_with_refs" => data([base], [ + pull_request(29, "Refs #10", "main", [ + { "sha" => "safe", "message" => "Safe first commit" }, + { "sha" => "fix-ref", "message" => "Fixes #10" } + ]) + ]), + "qualified_commit_fix_with_refs" => data([base], [ + pull_request(38, "Refs #10", "main", [ + { "sha" => "fix-ref", "message" => "Fixes proerror77/monday#10" } + ]) + ]), + "qualified_commit_with_closes" => data([base], [ + pull_request(40, "Closes #10", "main", [ + { "sha" => "qualified", "message" => "Fixes other/repo#10" } + ]) + ]), + "valid_qualified_commit_with_closes" => data([base], [ + pull_request(41, "Closes #10", "main", [ + { "sha" => "qualified", "message" => "Fixes example/repo#10" } + ]) + ]), + "commit_negated_resolve" => data([base], [ + pull_request(30, "Closes #10", "main", [ + { "sha" => "resolve", "message" => "This does not resolve #10" } + ]) + ]), + "commit_never_closes" => data([base], [ + pull_request(33, "Closes #10", "main", [ + { "sha" => "never", "message" => "This never closes #10" } + ]) + ]), + "commit_other_issue" => data([ + base, issue(52, %w[bug ready-for-human]) + ], [pull_request(31, "Closes #10", "main", [ + { "sha" => "closed", "message" => "Closed #52" } + ])]), + "commit_list_incomplete" => data([base], [incomplete_commits]) +} +File.write(ARGV.fetch(0), JSON.generate("cases" => cases)) +RUBY + +run_pass() { + local name="$1" + local expected="${2-}" + local output + output="$(ruby "$auditor" --fixture "$tmp_dir/cases.json" --case "$name")" + grep -Fq "PASS $name" <<<"$output" + if [[ -n "$expected" ]]; then + grep -Fq "$expected" <<<"$output" + fi +} + +run_fail() { + local name="$1" + local expected="$2" + local output audit_exit summary + summary="$tmp_dir/$name-summary.md" + set +e + output="$(ruby "$auditor" --fixture "$tmp_dir/cases.json" --case "$name" --summary "$summary" 2>&1)" + audit_exit=$? + set -e + test "$audit_exit" -eq 1 + grep -Fq "$expected" <<<"$output" + grep -Fq "$expected" "$summary" +} + +run_pass valid_refs +run_pass valid_closes +run_pass valid_none "automatic-linked-issue-closing: disabled (fixture)" +run_pass valid_needs_info +run_pass valid_wontfix +run_pass authorized_runtime_ready +run_pass authorized_runtime_ready_issue_form +run_pass valid_literal_escaped_newline +run_pass valid_qualified_commit_with_closes + +while IFS='|' read -r name expected; do + run_fail "$name" "$expected" +done <<'CASES' +missing_category|Issue #30: expected exactly one category label +conflicting_category|Issue #31: expected exactly one category label +missing_state|Issue #32: expected exactly one triage state label +conflicting_state|Issue #33: expected exactly one triage state label +literal_escaped_newline|Issue #34: body contains a literal escaped newline +short_literal_escaped_newline|Issue #61: body contains a literal escaped newline +mixed_literal_escaped_newline|Issue #60: body contains a literal escaped newline +tracking_agent_queue|Issue #35: tracking issues cannot use ready-for-agent +runtime_missing_control|Issue #36: runtime ready-for-agent is missing Runtime control: Candidate +runtime_open_blocker|Issue #37: runtime ready-for-agent has open native blocker #99 +active_missing_owner|Issue #40: active implementation requires exactly one assignee +active_multiple_owners|Issue #41: active implementation requires exactly one assignee +parent_mismatch|Issue #42: Parent summary references #456; native parent is #455 +blocker_mismatch|Issue #43: Blocked by summary references #78; native blocked-by is #77 +invalid_multiple_relationships|PR #21: expected exactly one visible issue relationship +negated_pr_closing|PR #24 body: negated closing phrase is forbidden +non_default_closes|PR #25: Closes #10 targets stack, not default branch main +runtime_closes|PR #26: runtime issue #50 cannot be closed by a pull request +tracking_closes|PR #27: tracking issue #51 cannot be closed by a pull request +pr_fix_with_refs|PR #28 body: closing keyword Fixes #10 requires visible Closes #10 +qualified_pr_fix_with_refs|PR #36 body: closing keyword Fixes proerror77/monday#10 requires visible Closes #10 +qualified_negated_pr_closing|PR #39 body: negated closing phrase is forbidden +modified_negated_pr_closing|PR #43 title: negated closing phrase is forbidden +pr_title_fix_with_refs|PR #34 title: closing keyword Fixes #10 requires visible Closes #10 +qualified_pr_title_fix_with_refs|PR #37 title: closing keyword Fixes proerror77/monday#10 requires visible Closes #10 +commit_fix_with_refs|PR #29 commit fix-ref: closing keyword Fixes #10 requires visible Closes #10 +qualified_commit_fix_with_refs|PR #38 commit fix-ref: closing keyword Fixes proerror77/monday#10 requires visible Closes #10 +qualified_commit_with_closes|PR #40 commit qualified: closing keyword Fixes other/repo#10 conflicts with visible Closes #10 +commit_negated_resolve|PR #30 commit resolve: negated closing phrase is forbidden +commit_never_closes|PR #33 commit never: negated closing phrase is forbidden +commit_other_issue|PR #31 commit closed: closing keyword Closed #52 conflicts with visible Closes #10 +commit_list_incomplete|PR #32: fetched 1 of 251 commit messages; audit cannot prove every commit safe +CASES + +set +e +parse_output="$(ruby "$auditor" --pr nope 2>&1)" +parse_exit=$? +summary_output="$(ruby "$auditor" --fixture "$tmp_dir/cases.json" --case valid_refs --summary "$tmp_dir/missing/summary.md" 2>&1)" +summary_exit=$? +set -e +test "$parse_exit" -eq 2 +grep -Fq "ERROR issue lifecycle audit" <<<"$parse_output" +test "$summary_exit" -eq 2 +grep -Fq "ERROR issue lifecycle audit" <<<"$summary_output" +if grep -Fq "PASS valid_refs" <<<"$summary_output"; then exit 1; fi + +live_stub_dir="$tmp_dir/live-stub" +mkdir -p "$live_stub_dir" +cat > "$live_stub_dir/gh" <<'EOF' +#!/usr/bin/env bash +path="${*: -1}" +case "$path" in + repos/example/repo) + echo '{"default_branch":"main"}' + ;; + repos/example/repo/issues\?state=open\&per_page=100\&page=1) + echo '[{"number":10,"title":"Fixture issue 10","body":"## Parent\n\nNone\n\n## Blocked by\n\nNone\n","labels":[{"name":"ready-for-agent"}],"assignees":[{"login":"agent"}],"issue_dependencies_summary":{"total_blocked_by":0}},{"number":88,"pull_request":{}}]' + ;; + repos/example/repo/pulls/88) + echo '{"number":88,"base":{"ref":"main"},"title":"Fixture PR 88","body":"## Issue relationship\n\nRefs #10\n\n## Focused validation\n\nFixture proof.\n","commits":1}' + ;; + repos/example/repo/pulls/88/commits\?per_page=100\&page=1) + echo '[{"sha":"safe","commit":{"message":"Safe commit"}}]' + ;; + repos/example/repo/issues/10) + echo '{"number":10,"title":"Fixture issue 10","body":"## Parent\n\nNone\n\n## Blocked by\n\nNone\n","labels":[{"name":"ready-for-agent"}],"assignees":[{"login":"agent"}],"issue_dependencies_summary":{"total_blocked_by":0}}' + ;; + repos/example/repo/issues/10/dependencies/blocked_by\?per_page=100\&page=1) + echo '[]' + ;; + *) + echo "unexpected gh api path: $path" >&2 + exit 1 + ;; +esac +EOF +chmod +x "$live_stub_dir/gh" + +set +e +live_output="$(PATH="$live_stub_dir:$PATH" ruby "$auditor" --repo example/repo --pr 88)" +live_exit=$? +set -e +test "$live_exit" -eq 1 +grep -Fq "FAIL PR #88" <<<"$live_output" +grep -Fq "Issue #10: expected exactly one category label" <<<"$live_output" + +echo "issue lifecycle audit fixtures: ok"