From 814a1c11be2808b5917472ef32ea1f86d4ac8e37 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:47:44 +0000 Subject: [PATCH 1/4] Initial plan From 9c16ee66da381db50ac8a6c512a9fc239c38cc63 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:58:50 +0000 Subject: [PATCH 2/4] test: add formal conformance registry invariants suite Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- ...config_conformance_registry_formal_test.go | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 pkg/workflow/awf_config_conformance_registry_formal_test.go diff --git a/pkg/workflow/awf_config_conformance_registry_formal_test.go b/pkg/workflow/awf_config_conformance_registry_formal_test.go new file mode 100644 index 00000000000..10484244263 --- /dev/null +++ b/pkg/workflow/awf_config_conformance_registry_formal_test.go @@ -0,0 +1,208 @@ +//go:build !integration + +package workflow + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type formalConformanceRegistryRow struct { + TestID string + Requirement string + TestFile string +} + +func formalConformanceRegistryBaselineRows() []formalConformanceRegistryRow { + return []formalConformanceRegistryRow{ + {TestID: "T-DR-001", Requirement: "§3.1 — required fields", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-002", Requirement: "§3.1 — drift_category enum", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-003", Requirement: "§3.1 — detected_at format", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-004", Requirement: "§3.1 — suggested_action non-empty", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-005", Requirement: "§3.1 — no additional properties", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-006", Requirement: "§7.5.1 — corrective PR trigger", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-007", Requirement: "§7.5.1 — SLA escalation trigger", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-008", Requirement: "§7.5.1 — corrective PR embeds records", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-009", Requirement: "§7.5.1 — empty list is valid", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-010", Requirement: "§7.2 Step 5 integration", TestFile: "pkg/workflow/awf_config_drift_test.go"}, + {TestID: "T-DR-SAFE-001", Requirement: "§8 item 1 — snapshot storage and freshness", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + {TestID: "T-DR-SAFE-002", Requirement: "§8 item 2 — retrieval warning", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + {TestID: "T-DR-SAFE-003", Requirement: "§8 item 3 — degraded-run safety", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + {TestID: "T-DR-SAFE-004", Requirement: "§8 item 4 — scheduled persistence", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + } +} + +func formalConformanceRegistryParseSeriesID(id string, prefix string) (int, bool) { + if !strings.HasPrefix(id, prefix) { + return 0, false + } + numeric := strings.TrimPrefix(id, prefix) + if len(numeric) < 3 { + return 0, false + } + for _, r := range numeric { + if r < '0' || r > '9' { + return 0, false + } + } + value, err := strconv.Atoi(numeric) + if err != nil { + return 0, false + } + return value, true +} + +func formalConformanceRegistryIsWellFormedFinalID(id string) bool { + if _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-"); ok { + return !strings.HasPrefix(id, "T-DR-SAFE-") + } + _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-SAFE-") + return ok +} + +func formalConformanceRegistryNextPlainID(rows []formalConformanceRegistryRow) string { + max := 0 + for _, row := range rows { + value, ok := formalConformanceRegistryParseSeriesID(row.TestID, "T-DR-") + if !ok || strings.HasPrefix(row.TestID, "T-DR-SAFE-") { + continue + } + if value > max { + max = value + } + } + return fmt.Sprintf("T-DR-%03d", max+1) +} + +func formalConformanceRegistryHasUniqueIDs(rows []formalConformanceRegistryRow) bool { + seen := make(map[string]struct{}, len(rows)) + for _, row := range rows { + if _, exists := seen[row.TestID]; exists { + return false + } + seen[row.TestID] = struct{}{} + } + return true +} + +func formalConformanceRegistryHasRequirementReference(row formalConformanceRegistryRow) bool { + return strings.TrimSpace(row.Requirement) != "" && strings.Contains(row.Requirement, "§") +} + +func formalConformanceRegistryHasImplementationFile(row formalConformanceRegistryRow) bool { + return strings.HasPrefix(row.TestFile, "pkg/workflow/") && strings.HasSuffix(row.TestFile, "_test.go") +} + +func formalConformanceRegistryRouteTestFile(spansDriftOutputAndSchema bool) string { + if spansDriftOutputAndSchema { + return "pkg/workflow/awf_config_drift_test.go" + } + return "pkg/workflow/awf_config_safeguards_formal_test.go" +} + +func formalConformanceRegistryHasSpecCrossReference(specIDs map[string]struct{}, id string) bool { + _, ok := specIDs[id] + return ok +} + +func formalConformanceRegistrySeriesDisjoint(id string) bool { + plain := strings.HasPrefix(id, "T-DR-") && !strings.HasPrefix(id, "T-DR-SAFE-") + safe := strings.HasPrefix(id, "T-DR-SAFE-") + return plain != safe +} + +func TestFormalConformanceRegistry_P1_TestIDMonotonicity(t *testing.T) { + next := formalConformanceRegistryNextPlainID(formalConformanceRegistryBaselineRows()) + assert.Equal(t, "T-DR-011", next) + + nextValue, ok := formalConformanceRegistryParseSeriesID(next, "T-DR-") + require.True(t, ok) + assert.Greater(t, nextValue, 10) +} + +func TestFormalConformanceRegistry_P1_EmptyRegistryStartsAtOne(t *testing.T) { + assert.Equal(t, "T-DR-001", formalConformanceRegistryNextPlainID(nil)) +} + +func TestFormalConformanceRegistry_P2_TestIDNoDuplicates(t *testing.T) { + rows := formalConformanceRegistryBaselineRows() + assert.True(t, formalConformanceRegistryHasUniqueIDs(rows)) + + rows = append(rows, formalConformanceRegistryRow{TestID: "T-DR-010", Requirement: "§x", TestFile: "pkg/workflow/awf_config_drift_test.go"}) + assert.False(t, formalConformanceRegistryHasUniqueIDs(rows)) +} + +func TestFormalConformanceRegistry_P3_TestIDFormatWellFormed(t *testing.T) { + valid := []string{"T-DR-001", "T-DR-010", "T-DR-1000", "T-DR-SAFE-001", "T-DR-SAFE-1234"} + invalid := []string{"t-dr-001", "T-DR-01", "T-DR-ABC", "T-DRSAFE-001", "T-DR-SAFE-1", "T-DR-SAFE-ABC"} + + for _, id := range valid { + assert.True(t, formalConformanceRegistryIsWellFormedFinalID(id), id) + } + for _, id := range invalid { + assert.False(t, formalConformanceRegistryIsWellFormedFinalID(id), id) + } +} + +func TestFormalConformanceRegistry_P4_PlaceholderIDRejectedAsFinal(t *testing.T) { + assert.False(t, formalConformanceRegistryIsWellFormedFinalID("T-DR-NNN")) +} + +func TestFormalConformanceRegistry_P5_RowHasRequirementReference(t *testing.T) { + for _, row := range formalConformanceRegistryBaselineRows() { + assert.True(t, formalConformanceRegistryHasRequirementReference(row), row.TestID) + } + assert.False(t, formalConformanceRegistryHasRequirementReference(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "required fields", TestFile: "pkg/workflow/awf_config_drift_test.go"})) +} + +func TestFormalConformanceRegistry_P6_RowHasImplementationFile(t *testing.T) { + for _, row := range formalConformanceRegistryBaselineRows() { + assert.True(t, formalConformanceRegistryHasImplementationFile(row), row.TestID) + } + assert.False(t, formalConformanceRegistryHasImplementationFile(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "§3.1", TestFile: ""})) +} + +func TestFormalConformanceRegistry_P7_SafeguardRowRoutingDecision(t *testing.T) { + assert.Equal(t, "pkg/workflow/awf_config_safeguards_formal_test.go", formalConformanceRegistryRouteTestFile(false)) + assert.Equal(t, "pkg/workflow/awf_config_drift_test.go", formalConformanceRegistryRouteTestFile(true)) +} + +func TestFormalConformanceRegistry_P8_SpecCrossReferenceRequired(t *testing.T) { + specIDs := map[string]struct{}{ + "T-DR-010": {}, + "T-DR-SAFE-004": {}, + } + assert.True(t, formalConformanceRegistryHasSpecCrossReference(specIDs, "T-DR-010")) + assert.False(t, formalConformanceRegistryHasSpecCrossReference(specIDs, "T-DR-011")) +} + +func TestFormalConformanceRegistry_P9_DriftSeriesVsSafeguardSeriesDisjoint(t *testing.T) { + assert.True(t, formalConformanceRegistrySeriesDisjoint("T-DR-010")) + assert.True(t, formalConformanceRegistrySeriesDisjoint("T-DR-SAFE-004")) + assert.False(t, formalConformanceRegistrySeriesDisjoint("T-DRX-010")) +} + +func TestFormalConformanceRegistry_EdgeCase_FourDigitRollover(t *testing.T) { + rows := []formalConformanceRegistryRow{{TestID: "T-DR-999", Requirement: "§x", TestFile: "pkg/workflow/awf_config_drift_test.go"}} + next := formalConformanceRegistryNextPlainID(rows) + assert.Equal(t, "T-DR-1000", next) + assert.True(t, formalConformanceRegistryIsWellFormedFinalID(next)) +} + +func TestFormalConformanceRegistry_EdgeCase_SafeguardOnlyRegistryDoesNotAffectPlainSeries(t *testing.T) { + rows := []formalConformanceRegistryRow{ + {TestID: "T-DR-SAFE-001", Requirement: "§8", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + {TestID: "T-DR-SAFE-004", Requirement: "§8", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + } + assert.Equal(t, "T-DR-001", formalConformanceRegistryNextPlainID(rows)) +} + +func TestFormalConformanceRegistry_EdgeCase_MissingImplementationFileIsInvalid(t *testing.T) { + row := formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "§3.1", TestFile: ""} + assert.False(t, formalConformanceRegistryHasImplementationFile(row)) +} From 5b23360548702c18428533ceee1a4eb1b39bd71f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:03:26 +0000 Subject: [PATCH 3/4] test: tighten formal conformance registry id checks Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- ...config_conformance_registry_formal_test.go | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/workflow/awf_config_conformance_registry_formal_test.go b/pkg/workflow/awf_config_conformance_registry_formal_test.go index 10484244263..4322f5380ca 100644 --- a/pkg/workflow/awf_config_conformance_registry_formal_test.go +++ b/pkg/workflow/awf_config_conformance_registry_formal_test.go @@ -57,19 +57,27 @@ func formalConformanceRegistryParseSeriesID(id string, prefix string) (int, bool return value, true } +func formalConformanceRegistryParsePlainID(id string) (int, bool) { + if strings.HasPrefix(id, "T-DR-SAFE-") { + return 0, false + } + return formalConformanceRegistryParseSeriesID(id, "T-DR-") +} + func formalConformanceRegistryIsWellFormedFinalID(id string) bool { - if _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-"); ok { - return !strings.HasPrefix(id, "T-DR-SAFE-") + if strings.HasPrefix(id, "T-DR-SAFE-") { + _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-SAFE-") + return ok } - _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-SAFE-") + _, ok := formalConformanceRegistryParsePlainID(id) return ok } func formalConformanceRegistryNextPlainID(rows []formalConformanceRegistryRow) string { max := 0 for _, row := range rows { - value, ok := formalConformanceRegistryParseSeriesID(row.TestID, "T-DR-") - if !ok || strings.HasPrefix(row.TestID, "T-DR-SAFE-") { + value, ok := formalConformanceRegistryParsePlainID(row.TestID) + if !ok { continue } if value > max { @@ -120,9 +128,9 @@ func TestFormalConformanceRegistry_P1_TestIDMonotonicity(t *testing.T) { next := formalConformanceRegistryNextPlainID(formalConformanceRegistryBaselineRows()) assert.Equal(t, "T-DR-011", next) - nextValue, ok := formalConformanceRegistryParseSeriesID(next, "T-DR-") + nextValue, ok := formalConformanceRegistryParsePlainID(next) require.True(t, ok) - assert.Greater(t, nextValue, 10) + assert.Equal(t, 11, nextValue) } func TestFormalConformanceRegistry_P1_EmptyRegistryStartsAtOne(t *testing.T) { @@ -139,7 +147,7 @@ func TestFormalConformanceRegistry_P2_TestIDNoDuplicates(t *testing.T) { func TestFormalConformanceRegistry_P3_TestIDFormatWellFormed(t *testing.T) { valid := []string{"T-DR-001", "T-DR-010", "T-DR-1000", "T-DR-SAFE-001", "T-DR-SAFE-1234"} - invalid := []string{"t-dr-001", "T-DR-01", "T-DR-ABC", "T-DRSAFE-001", "T-DR-SAFE-1", "T-DR-SAFE-ABC"} + invalid := []string{"t-dr-001", "T-DR-01", "T-DR-ABC", "T-DRSAFE-001", "T-DR-SAFE-1", "T-DR-SAFE-01", "T-DR-SAFE-ABC"} for _, id := range valid { assert.True(t, formalConformanceRegistryIsWellFormedFinalID(id), id) From f2513c4cb984b7ef64954f85215c9560fab953c1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:38:58 +0000 Subject: [PATCH 4/4] test: validate documented conformance registry Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- ...config_conformance_registry_formal_test.go | 98 +++++++++++++------ specs/awf-config-sources-compliance/README.md | 36 +++---- specs/awf-config-sources-spec.md | 12 ++- 3 files changed, 98 insertions(+), 48 deletions(-) diff --git a/pkg/workflow/awf_config_conformance_registry_formal_test.go b/pkg/workflow/awf_config_conformance_registry_formal_test.go index 4322f5380ca..17b8b562245 100644 --- a/pkg/workflow/awf_config_conformance_registry_formal_test.go +++ b/pkg/workflow/awf_config_conformance_registry_formal_test.go @@ -4,6 +4,9 @@ package workflow import ( "fmt" + "os" + "path/filepath" + "runtime" "strconv" "strings" "testing" @@ -18,23 +21,43 @@ type formalConformanceRegistryRow struct { TestFile string } -func formalConformanceRegistryBaselineRows() []formalConformanceRegistryRow { - return []formalConformanceRegistryRow{ - {TestID: "T-DR-001", Requirement: "§3.1 — required fields", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-002", Requirement: "§3.1 — drift_category enum", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-003", Requirement: "§3.1 — detected_at format", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-004", Requirement: "§3.1 — suggested_action non-empty", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-005", Requirement: "§3.1 — no additional properties", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-006", Requirement: "§7.5.1 — corrective PR trigger", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-007", Requirement: "§7.5.1 — SLA escalation trigger", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-008", Requirement: "§7.5.1 — corrective PR embeds records", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-009", Requirement: "§7.5.1 — empty list is valid", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-010", Requirement: "§7.2 Step 5 integration", TestFile: "pkg/workflow/awf_config_drift_test.go"}, - {TestID: "T-DR-SAFE-001", Requirement: "§8 item 1 — snapshot storage and freshness", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, - {TestID: "T-DR-SAFE-002", Requirement: "§8 item 2 — retrieval warning", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, - {TestID: "T-DR-SAFE-003", Requirement: "§8 item 3 — degraded-run safety", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, - {TestID: "T-DR-SAFE-004", Requirement: "§8 item 4 — scheduled persistence", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, +func formalConformanceRegistryRepositoryRoot(t *testing.T) string { + t.Helper() + + _, file, _, ok := runtime.Caller(0) + require.True(t, ok) + return filepath.Clean(filepath.Join(filepath.Dir(file), "../..")) +} + +func formalConformanceRegistryReadFile(t *testing.T, relativePath string) string { + t.Helper() + + content, err := os.ReadFile(filepath.Join(formalConformanceRegistryRepositoryRoot(t), relativePath)) + require.NoError(t, err) + return string(content) +} + +func formalConformanceRegistryBaselineRows(t *testing.T) []formalConformanceRegistryRow { + t.Helper() + + content := formalConformanceRegistryReadFile(t, "specs/awf-config-sources-compliance/README.md") + var rows []formalConformanceRegistryRow + for line := range strings.SplitSeq(content, "\n") { + if !strings.HasPrefix(line, "| T-DR-") { + continue + } + + cells := strings.Split(line, "|") + require.Len(t, cells, 6, "registry row: %s", line) + rows = append(rows, formalConformanceRegistryRow{ + TestID: strings.TrimSpace(cells[1]), + Requirement: strings.TrimSpace(cells[2]), + TestFile: strings.Trim(strings.TrimSpace(cells[4]), "`"), + }) } + + require.NotEmpty(t, rows) + return rows } func formalConformanceRegistryParseSeriesID(id string, prefix string) (int, bool) { @@ -113,9 +136,27 @@ func formalConformanceRegistryRouteTestFile(spansDriftOutputAndSchema bool) stri return "pkg/workflow/awf_config_safeguards_formal_test.go" } -func formalConformanceRegistryHasSpecCrossReference(specIDs map[string]struct{}, id string) bool { - _, ok := specIDs[id] - return ok +func formalConformanceRegistryHasSpecCrossReference(specContent, id string) bool { + for offset := 0; ; { + index := strings.Index(specContent[offset:], id) + if index < 0 { + return false + } + index += offset + end := index + len(id) + if (index == 0 || !formalConformanceRegistryIDCharacter(specContent[index-1])) && + (end == len(specContent) || !formalConformanceRegistryIDCharacter(specContent[end])) { + return true + } + offset = end + } +} + +func formalConformanceRegistryIDCharacter(character byte) bool { + return character >= 'A' && character <= 'Z' || + character >= 'a' && character <= 'z' || + character >= '0' && character <= '9' || + character == '-' } func formalConformanceRegistrySeriesDisjoint(id string) bool { @@ -125,7 +166,7 @@ func formalConformanceRegistrySeriesDisjoint(id string) bool { } func TestFormalConformanceRegistry_P1_TestIDMonotonicity(t *testing.T) { - next := formalConformanceRegistryNextPlainID(formalConformanceRegistryBaselineRows()) + next := formalConformanceRegistryNextPlainID(formalConformanceRegistryBaselineRows(t)) assert.Equal(t, "T-DR-011", next) nextValue, ok := formalConformanceRegistryParsePlainID(next) @@ -138,7 +179,7 @@ func TestFormalConformanceRegistry_P1_EmptyRegistryStartsAtOne(t *testing.T) { } func TestFormalConformanceRegistry_P2_TestIDNoDuplicates(t *testing.T) { - rows := formalConformanceRegistryBaselineRows() + rows := formalConformanceRegistryBaselineRows(t) assert.True(t, formalConformanceRegistryHasUniqueIDs(rows)) rows = append(rows, formalConformanceRegistryRow{TestID: "T-DR-010", Requirement: "§x", TestFile: "pkg/workflow/awf_config_drift_test.go"}) @@ -162,15 +203,16 @@ func TestFormalConformanceRegistry_P4_PlaceholderIDRejectedAsFinal(t *testing.T) } func TestFormalConformanceRegistry_P5_RowHasRequirementReference(t *testing.T) { - for _, row := range formalConformanceRegistryBaselineRows() { + for _, row := range formalConformanceRegistryBaselineRows(t) { assert.True(t, formalConformanceRegistryHasRequirementReference(row), row.TestID) } assert.False(t, formalConformanceRegistryHasRequirementReference(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "required fields", TestFile: "pkg/workflow/awf_config_drift_test.go"})) } func TestFormalConformanceRegistry_P6_RowHasImplementationFile(t *testing.T) { - for _, row := range formalConformanceRegistryBaselineRows() { + for _, row := range formalConformanceRegistryBaselineRows(t) { assert.True(t, formalConformanceRegistryHasImplementationFile(row), row.TestID) + assert.FileExists(t, filepath.Join(formalConformanceRegistryRepositoryRoot(t), row.TestFile), row.TestID) } assert.False(t, formalConformanceRegistryHasImplementationFile(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "§3.1", TestFile: ""})) } @@ -181,12 +223,12 @@ func TestFormalConformanceRegistry_P7_SafeguardRowRoutingDecision(t *testing.T) } func TestFormalConformanceRegistry_P8_SpecCrossReferenceRequired(t *testing.T) { - specIDs := map[string]struct{}{ - "T-DR-010": {}, - "T-DR-SAFE-004": {}, + specContent := formalConformanceRegistryReadFile(t, "specs/awf-config-sources-spec.md") + for _, row := range formalConformanceRegistryBaselineRows(t) { + assert.True(t, formalConformanceRegistryHasSpecCrossReference(specContent, row.TestID), row.TestID) } - assert.True(t, formalConformanceRegistryHasSpecCrossReference(specIDs, "T-DR-010")) - assert.False(t, formalConformanceRegistryHasSpecCrossReference(specIDs, "T-DR-011")) + + assert.False(t, formalConformanceRegistryHasSpecCrossReference(specContent, "T-DR-011")) } func TestFormalConformanceRegistry_P9_DriftSeriesVsSafeguardSeriesDisjoint(t *testing.T) { diff --git a/specs/awf-config-sources-compliance/README.md b/specs/awf-config-sources-compliance/README.md index d8fd1ad43d9..0de78c3515f 100644 --- a/specs/awf-config-sources-compliance/README.md +++ b/specs/awf-config-sources-compliance/README.md @@ -12,18 +12,18 @@ defined in §3.1 of the specification for structured drift output. The following test IDs cover the `DriftRecord` schema and its usage requirements from §3.1 and §7.5. -| Test ID | Requirement | Description | -|---------|-------------|-------------| -| T-DR-001 | §3.1 — required fields | `DriftRecord` MUST include `property_path`, `drift_category`, `suggested_action`, and `detected_at`; records missing any required field are invalid and MUST be rejected. | -| T-DR-002 | §3.1 — `drift_category` enum | `drift_category` MUST be one of `missing_in_ghaw`, `missing_in_schema`, or `spec_mismatch`; any other value is invalid. | -| T-DR-003 | §3.1 — `detected_at` format | `detected_at` MUST be a valid ISO 8601 UTC timestamp; non-conforming values MUST be rejected. | -| T-DR-004 | §3.1 — `suggested_action` non-empty | `suggested_action` MUST NOT be empty (`minLength: 1`); an empty string MUST be rejected. | -| T-DR-005 | §3.1 — no additional properties | `DriftRecord` objects MUST NOT include properties beyond the four required fields; additional properties MUST be rejected. | -| T-DR-006 | §7.5.1 — corrective PR trigger | When any `DriftRecord` in the output list has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation MUST open a corrective PR (CR-05). | -| T-DR-007 | §7.5.1 — SLA escalation trigger | When CR-06 SLA window is exceeded and `DriftRecord` items with actionable categories are present, an escalation issue MUST be opened or updated. | -| T-DR-008 | §7.5.1 — corrective PR embeds records | The corrective PR description MUST embed the full `DriftRecord` list as JSON. | -| T-DR-009 | §7.5.1 — empty list is valid | An empty `DriftRecord` list (no drift detected) is a valid output and MUST NOT trigger corrective PR or escalation actions. | -| T-DR-010 | §7.2 Step 5 integration | The drift detection procedure Step 5 MUST produce a list of zero or more `DriftRecord` objects; the output format MUST be a JSON array conforming to the §3.1 schema. | +| Test ID | Requirement | Description | Implementation file | +|---------|-------------|-------------|---------------------| +| T-DR-001 | §3.1 — required fields | `DriftRecord` MUST include `property_path`, `drift_category`, `suggested_action`, and `detected_at`; records missing any required field are invalid and MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-002 | §3.1 — `drift_category` enum | `drift_category` MUST be one of `missing_in_ghaw`, `missing_in_schema`, or `spec_mismatch`; any other value is invalid. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-003 | §3.1 — `detected_at` format | `detected_at` MUST be a valid ISO 8601 UTC timestamp; non-conforming values MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-004 | §3.1 — `suggested_action` non-empty | `suggested_action` MUST NOT be empty (`minLength: 1`); an empty string MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-005 | §3.1 — no additional properties | `DriftRecord` objects MUST NOT include properties beyond the four required fields; additional properties MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-006 | §7.5.1 — corrective PR trigger | When any `DriftRecord` in the output list has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation MUST open a corrective PR (CR-05). | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-007 | §7.5.1 — SLA escalation trigger | When CR-06 SLA window is exceeded and `DriftRecord` items with actionable categories are present, an escalation issue MUST be opened or updated. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-008 | §7.5.1 — corrective PR embeds records | The corrective PR description MUST embed the full `DriftRecord` list as JSON. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-009 | §7.5.1 — empty list is valid | An empty `DriftRecord` list (no drift detected) is a valid output and MUST NOT trigger corrective PR or escalation actions. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-010 | §7.2 Step 5 integration | The drift detection procedure Step 5 MUST produce a list of zero or more `DriftRecord` objects; the output format MUST be a JSON array conforming to the §3.1 schema. | `pkg/workflow/awf_config_drift_test.go` | --- @@ -31,12 +31,12 @@ The following test IDs cover the `DriftRecord` schema and its usage requirements The following test IDs cover the unavailable-source safeguards from §8. -| Test ID | Requirement | Description | -|---------|-------------|-------------| -| T-DR-SAFE-001 | §8 item 1 — snapshot storage and freshness | Every invocation MUST select the stable path for its runner type, expire snapshots older than 168 hours, mark expired-snapshot runs degraded, and SHOULD delete snapshots older than 14 days. | -| T-DR-SAFE-002 | §8 item 2 — retrieval warning | A canonical-source retrieval failure SHOULD identify the failing source paths and UTC timestamp. | -| T-DR-SAFE-003 | §8 item 3 — degraded-run safety | An unavailable or expired canonical source MUST mark the run degraded and MUST prevent destructive validation actions. | -| T-DR-SAFE-004 | §8 item 4 — scheduled persistence | A tracking issue SHOULD be opened or updated only when unavailability persists through the next scheduled cron invocation; manual and ad hoc runs do not advance the threshold. | +| Test ID | Requirement | Description | Implementation file | +|---------|-------------|-------------|---------------------| +| T-DR-SAFE-001 | §8 item 1 — snapshot storage and freshness | Every invocation MUST select the stable path for its runner type, expire snapshots older than 168 hours, mark expired-snapshot runs degraded, and SHOULD delete snapshots older than 14 days. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-002 | §8 item 2 — retrieval warning | A canonical-source retrieval failure SHOULD identify the failing source paths and UTC timestamp. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-003 | §8 item 3 — degraded-run safety | An unavailable or expired canonical source MUST mark the run degraded and MUST prevent destructive validation actions. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-004 | §8 item 4 — scheduled persistence | A tracking issue SHOULD be opened or updated only when unavailability persists through the next scheduled cron invocation; manual and ad hoc runs do not advance the threshold. | `pkg/workflow/awf_config_safeguards_formal_test.go` | --- diff --git a/specs/awf-config-sources-spec.md b/specs/awf-config-sources-spec.md index 9fd77a1c6f4..85fd5828cf0 100644 --- a/specs/awf-config-sources-spec.md +++ b/specs/awf-config-sources-spec.md @@ -110,6 +110,14 @@ A `DriftRecord` represents a single detected schema drift item. All automation a | `suggested_action` | `string` | **MUST** | Actionable remediation text; **MUST NOT** be empty | `pkg/workflow/awf_config_drift_formal_test.go` (`FormalDriftRecord.SuggestedAction`, `formalDriftRecordStructuralValidity`); production emission target: `pkg/workflow/` drift detection logic | | `detected_at` | `string` (ISO 8601) | **MUST** | UTC timestamp of detection; filesystem-safe format **SHOULD** use `YYYY-MM-DDTHH:MM:SSZ` | `pkg/workflow/awf_config_drift_formal_test.go` (`FormalDriftRecord.DetectedAt`); production emission target: `pkg/workflow/` drift detection logic | +The conformance fixture index assigns the following test IDs to the `DriftRecord` schema requirements: + +- Required fields: T-DR-001 +- `drift_category` enum: T-DR-002 +- `detected_at` format: T-DR-003 +- `suggested_action` non-empty: T-DR-004 +- No additional properties: T-DR-005 + ## 4. Required coverage checks When updating AWF config generation, schema sync, or validation in gh-aw, agents MUST verify: @@ -200,7 +208,7 @@ Drift detection MUST be triggered when: - **Missing in schema**: `gh-aw` generates a field not present in either schema. - **Spec mismatch**: CLI mapping in `gh-aw` disagrees with the normative spec description. -5. **Produce a drift report** listing: +5. **Produce a drift report** (T-DR-010) listing: - Each drifted property path (e.g., `apiProxy.anthropicAutoCache`). - Drift category (missing in gh-aw / missing in schema / spec mismatch). - Suggested corrective action (add coverage, open PR, update spec). @@ -279,7 +287,7 @@ A `DriftRecord` represents a single detected schema drift item produced by the d #### 7.5.1 Usage -The drift detection procedure (Section 7.2, Step 5) **MUST** produce a list of zero or more `DriftRecord` objects (schema: Section 3.1). When any record has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation **MUST** open a corrective PR (CR-05) and, if the SLA window is exceeded, an escalation issue (CR-06). The corrective PR description **MUST** embed the full `DriftRecord` list as JSON. +The drift detection procedure (Section 7.2, Step 5) **MUST** produce a list of zero or more `DriftRecord` objects (schema: Section 3.1; T-DR-009). When any record has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation **MUST** open a corrective PR (CR-05; T-DR-006) and, if the SLA window is exceeded, an escalation issue (CR-06; T-DR-007). The corrective PR description **MUST** embed the full `DriftRecord` list as JSON (T-DR-008). **Example output (Step 5 of the drift detection procedure):**