diff --git a/tools/skill-evals/evals/release-verify-rc/README.md b/tools/skill-evals/evals/release-verify-rc/README.md index d1e931656..423ead0f7 100644 --- a/tools/skill-evals/evals/release-verify-rc/README.md +++ b/tools/skill-evals/evals/release-verify-rc/README.md @@ -15,7 +15,7 @@ Behavioural eval suite for the | `step-3-verify-checksums` | Step 3 — Verify checksums | 2 | Checksum classification (PASS/MISMATCH/MISSING-DIGEST), deprecated md5 detection | | `step-5-notice-license` | Step 5 — NOTICE/LICENSE presence | 2 | File presence (PASS/WARN/FAIL), diff-lines count, diff summary | | `step-6-binary-exclusion` | Step 6 — Binary exclusion check | 2 | Prohibited-binary detection (PASS/FAIL), expected-binary classification | -| `step-7-version-consistency` | Step 7 — Version string consistency | 2 | Exact version match across manifest files (PASS/FAIL) | +| `step-8-version-consistency` | Step 8 — Version string consistency | 2 | Exact version match across manifest files (PASS/FAIL) | Total: **13 cases** across 6 step suites. @@ -34,7 +34,7 @@ uv run --project tools/skill-evals skill-eval --cli tools/skill-evals/evals/rele ## Grading methodology -Steps 0, 2, 3, 5, 6, and 7 all emit structured JSON. Cases use +Steps 0, 2, 3, 5, 6, and 8 all emit structured JSON. Cases use `expected.json` for exact-field grading and `output-spec.md` to document the allowed schema. @@ -61,5 +61,5 @@ are ignored; fields present in `expected.json` must match exactly `"WARN"` for material diffs; `"PASS"` for version-string-only diffs. - **Step 6**: `status` must be `"FAIL"` when `prohibited_found` is non-empty. -- **Step 7**: `status` must be `"FAIL"` for any `match: false` or +- **Step 8**: `status` must be `"FAIL"` for any `match: false` or `extracted: null`; dev/snapshot suffixes are always `match: false`. diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/step-config.json b/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/step-config.json deleted file mode 100644 index 1c40f25f1..000000000 --- a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/step-config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "skill_md": "skills/release-verify-rc/SKILL.md", - "step_heading": "## Step 7 — Version string consistency" -} diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-1-version-consistent/expected.json b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-1-version-consistent/expected.json similarity index 100% rename from tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-1-version-consistent/expected.json rename to tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-1-version-consistent/expected.json diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-1-version-consistent/report.md b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-1-version-consistent/report.md similarity index 100% rename from tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-1-version-consistent/report.md rename to tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-1-version-consistent/report.md diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-2-version-mismatch/expected.json b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-2-version-mismatch/expected.json similarity index 100% rename from tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-2-version-mismatch/expected.json rename to tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-2-version-mismatch/expected.json diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-2-version-mismatch/report.md b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-2-version-mismatch/report.md similarity index 100% rename from tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/case-2-version-mismatch/report.md rename to tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/case-2-version-mismatch/report.md diff --git a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/output-spec.md similarity index 96% rename from tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/output-spec.md rename to tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/output-spec.md index 63450ca2a..27d6737a4 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-7-version-consistency/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/output-spec.md @@ -1,7 +1,7 @@ -# Step 7 output specification +# Step 8 output specification The model must return ONLY valid JSON matching this schema: diff --git a/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/step-config.json b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/step-config.json new file mode 100644 index 000000000..bd1aed98e --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-8-version-consistency/fixtures/step-config.json @@ -0,0 +1,4 @@ +{ + "skill_md": "skills/release-verify-rc/SKILL.md", + "step_heading": "## Step 8 — Version string consistency" +}