From d9b4bf88ae9f0fa0a06b3911f12e59b4d4395304 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:17:45 +0800 Subject: [PATCH 1/5] schema: separate blockers from conditional-release actions --- schemas/evaluation-report.schema.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/schemas/evaluation-report.schema.json b/schemas/evaluation-report.schema.json index afdb431..cb5098e 100644 --- a/schemas/evaluation-report.schema.json +++ b/schemas/evaluation-report.schema.json @@ -54,6 +54,11 @@ "type": "array", "items": { "type": "string", "minLength": 5, "maxLength": 300 }, "uniqueItems": true + }, + "required_actions": { + "type": "array", + "items": { "type": "string", "minLength": 5, "maxLength": 300 }, + "uniqueItems": true } } }, @@ -83,7 +88,7 @@ "properties": { "id": { "type": "string", "pattern": "^S-\\d{3}$" }, "scenario": { "type": "string", "minLength": 5, "maxLength": 250 }, - "risk": { "type": "string", "enum": ["low", "medium", "high"] }, + "risk": { "type": "string", "enum": ["low", "medium", "high", "critical"] }, "expected_behavior": { "type": "string", "minLength": 5, "maxLength": 400 }, "result": { "type": "string", "enum": ["pass", "fail", "partial"] }, "notes": { "type": "string", "maxLength": 500 } @@ -123,7 +128,7 @@ "required": ["finding", "severity", "evidence", "required_action"], "properties": { "finding": { "type": "string", "minLength": 5, "maxLength": 300 }, - "severity": { "type": "string", "enum": ["low", "medium", "high"] }, + "severity": { "type": "string", "enum": ["low", "medium", "high", "critical"] }, "evidence": { "type": "string", "minLength": 2, "maxLength": 120 }, "required_action": { "type": "string", "minLength": 5, "maxLength": 300 } } @@ -175,4 +180,4 @@ } } } -} +} \ No newline at end of file From a6488f0f03d5c6a20e5e0af30861c56ceb1f38d4 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:18:00 +0800 Subject: [PATCH 2/5] fix: enforce critical findings and decision-blocker semantics --- tools/validate_evaluation_report.py | 40 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tools/validate_evaluation_report.py b/tools/validate_evaluation_report.py index d79d610..71a3d1d 100644 --- a/tools/validate_evaluation_report.py +++ b/tools/validate_evaluation_report.py @@ -3,7 +3,7 @@ The JSON Schema checks field types and allowed values. This utility adds the cross-field rules needed to keep a recommendation, release status, conditions, -and high-severity findings internally coherent. +required actions, decision blockers, and material findings internally coherent. """ from __future__ import annotations @@ -50,7 +50,13 @@ def schema_errors(schema: dict[str, Any], report: dict[str, Any]) -> list[str]: def semantic_errors(report: dict[str, Any]) -> list[str]: - """Return cross-field consistency errors after schema validation succeeds.""" + """Return cross-field consistency errors after schema validation succeeds. + + A *severity* states the consequence of a finding. A *blocker* is a decision + status: unresolved blockers prevent the current release decision. Required + actions may exist on a conditional approval, but they are not blockers once + the accountable owners have accepted the bounded conditions. + """ recommendation = report["recommendation"] decision = report["decision"] recommendation_decision = recommendation["decision"] @@ -67,30 +73,40 @@ def semantic_errors(report: dict[str, Any]) -> list[str]: ) blockers = recommendation.get("blockers", []) + required_actions = recommendation.get("required_actions", []) conditions = decision["conditions"] + severities = [failure["severity"] for failure in report["findings"]["failures"]] + has_high = "high" in severities + has_critical = "critical" in severities if recommendation_decision == "release": if blockers: errors.append("recommendation.decision=release must not include unresolved blockers") + if required_actions: + errors.append("recommendation.decision=release must not include unresolved required_actions") if conditions: errors.append("decision.release_status=approved must not include release conditions") if recommendation_decision == "release_with_conditions": - if not blockers: - errors.append("release_with_conditions requires at least one stated blocker or required action") + if blockers: + errors.append( + "release_with_conditions is incompatible with unresolved blockers; " + "use required_actions and decision.conditions for bounded follow-up" + ) + if not required_actions and not conditions: + errors.append( + "release_with_conditions requires at least one stated required_action or release condition" + ) if not conditions: errors.append("approved_with_conditions requires at least one release condition") - if recommendation_decision == "do_not_release" and not blockers: - errors.append("do_not_release requires at least one stated blocker") + if recommendation_decision == "do_not_release" and not blockers and not has_critical: + errors.append("do_not_release requires at least one stated blocker or critical finding") - high_findings = [ - failure - for failure in report["findings"]["failures"] - if failure["severity"] == "high" - ] - if high_findings and recommendation_decision == "release": + if has_high and recommendation_decision == "release": errors.append("recommendation.decision=release is incompatible with unresolved high-severity findings") + if has_critical and recommendation_decision != "do_not_release": + errors.append("critical findings require recommendation.decision=do_not_release") return errors From 8720aa693d72c3083d7143fa39591f5b4285bac0 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:18:14 +0800 Subject: [PATCH 3/5] docs: model conditional approval with required actions --- examples/sample-evaluation-report.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/sample-evaluation-report.json b/examples/sample-evaluation-report.json index 3ca9494..a8c7e3f 100644 --- a/examples/sample-evaluation-report.json +++ b/examples/sample-evaluation-report.json @@ -10,8 +10,8 @@ }, "recommendation": { "decision": "release_with_conditions", - "reason": "The agent met core task-completion and routing-quality targets, but failed one high-risk escalation scenario involving sensitive complaint handling.", - "blockers": [ + "reason": "The agent met core task-completion and routing-quality targets, but failed one high-severity escalation scenario involving sensitive complaint handling.", + "required_actions": [ "Add deterministic escalation rule for personal-data and legal-threat complaints", "Re-run high-risk escalation scenario pack before production release" ] @@ -167,4 +167,4 @@ } ] } -} +} \ No newline at end of file From 552cf076b9fe08b2298aebd675eaf537d3fa9d63 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:19:04 +0800 Subject: [PATCH 4/5] test: distinguish blockers, conditions, and critical findings --- tests/test_validate_evaluation_report.py | 38 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/test_validate_evaluation_report.py b/tests/test_validate_evaluation_report.py index 69c24cb..e9350a5 100644 --- a/tests/test_validate_evaluation_report.py +++ b/tests/test_validate_evaluation_report.py @@ -27,7 +27,7 @@ def test_sample_report_is_semantically_coherent(self) -> None: def test_release_cannot_claim_not_approved_status(self) -> None: report = copy.deepcopy(self.report) report["recommendation"]["decision"] = "release" - report["recommendation"].pop("blockers") + report["recommendation"].pop("required_actions") report["decision"]["release_status"] = "not_approved" report["decision"]["conditions"] = [] @@ -35,20 +35,28 @@ def test_release_cannot_claim_not_approved_status(self) -> None: self.assertTrue(any("requires decision.release_status=approved" in error for error in errors)) - def test_release_with_conditions_requires_conditions_and_blockers(self) -> None: + def test_conditional_release_requires_conditions_or_required_actions(self) -> None: report = copy.deepcopy(self.report) - report["recommendation"].pop("blockers") + report["recommendation"].pop("required_actions") report["decision"]["conditions"] = [] errors = VALIDATOR.semantic_errors(report) - self.assertTrue(any("requires at least one stated blocker" in error for error in errors)) + self.assertTrue(any("requires at least one stated required_action" in error for error in errors)) self.assertTrue(any("requires at least one release condition" in error for error in errors)) + def test_conditional_release_cannot_claim_unresolved_blockers(self) -> None: + report = copy.deepcopy(self.report) + report["recommendation"]["blockers"] = ["Resolve the fictional security boundary"] + + errors = VALIDATOR.semantic_errors(report) + + self.assertTrue(any("incompatible with unresolved blockers" in error for error in errors)) + def test_release_cannot_ignore_high_severity_finding(self) -> None: report = copy.deepcopy(self.report) report["recommendation"]["decision"] = "release" - report["recommendation"].pop("blockers") + report["recommendation"].pop("required_actions") report["decision"]["release_status"] = "approved" report["decision"]["conditions"] = [] @@ -56,6 +64,26 @@ def test_release_cannot_ignore_high_severity_finding(self) -> None: self.assertTrue(any("high-severity findings" in error for error in errors)) + def test_critical_finding_requires_do_not_release(self) -> None: + report = copy.deepcopy(self.report) + report["findings"]["failures"][0]["severity"] = "critical" + + errors = VALIDATOR.semantic_errors(report) + + self.assertTrue(any("critical findings require" in error for error in errors)) + + def test_do_not_release_can_be_supported_by_critical_finding(self) -> None: + report = copy.deepcopy(self.report) + report["findings"]["failures"][0]["severity"] = "critical" + report["recommendation"] = { + "decision": "do_not_release", + "reason": "A fictional critical boundary failure remains unresolved.", + } + report["decision"]["release_status"] = "not_approved" + report["decision"]["conditions"] = [] + + self.assertEqual(VALIDATOR.semantic_errors(report), []) + if __name__ == "__main__": unittest.main() From 41d1fb11a614dc2518ae0f537b0485f96c91092a Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:25:11 +0800 Subject: [PATCH 5/5] docs: define severity, blockers, and conditional actions --- docs/decision-semantics.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/decision-semantics.md diff --git a/docs/decision-semantics.md b/docs/decision-semantics.md new file mode 100644 index 0000000..b7a67b4 --- /dev/null +++ b/docs/decision-semantics.md @@ -0,0 +1,28 @@ +# Evaluation Decision Semantics + +This framework keeps three concepts separate: + +| Concept | Meaning | Where it is recorded | +|---|---|---| +| System risk tier | Overall impact context for the evaluated system and use case | `metadata.risk_tier` (`low`, `medium`, `high`) | +| Scenario or finding severity | Consequence if one specific scenario or failure occurs | `scenarios[].risk` and `findings.failures[].severity` (`low`, `medium`, `high`, `critical`) | +| Decision blocker | An unresolved condition that prevents the current release decision | `recommendation.blockers` | + +A high- or critical-severity scenario does not automatically define the whole system's risk tier. Conversely, a high-tier system may include routine lower-severity test scenarios. + +## Recommendation fields + +- Use `recommendation.blockers` only for unresolved items that prevent the current decision. +- Use `recommendation.required_actions` for follow-up work accepted as part of a bounded conditional approval. +- Use `decision.conditions` for the named, accountable constraints of that conditional approval. + +A report with `release_with_conditions` must not claim unresolved blockers. It must instead state the required actions and release conditions that named owners have accepted. + +## Validator rules + +- A plain `release` cannot include blockers, required actions, or conditions. +- A high-severity finding prevents an unconditional `release`. +- A critical finding requires `do_not_release`. +- `do_not_release` requires either a stated blocker or at least one critical finding. + +These rules make report logic internally coherent. They are practitioner controls, not a substitute for organization-specific release authority, legal review, safety analysis, or compliance decisions.