diff --git a/pkg/workflow/awf_config_conformance_registry_formal_test.go b/pkg/workflow/awf_config_conformance_registry_formal_test.go new file mode 100644 index 00000000000..17b8b562245 --- /dev/null +++ b/pkg/workflow/awf_config_conformance_registry_formal_test.go @@ -0,0 +1,258 @@ +//go:build !integration + +package workflow + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type formalConformanceRegistryRow struct { + TestID string + Requirement string + TestFile string +} + +func formalConformanceRegistryRepositoryRoot(t *testing.T) string { + t.Helper() + + _, file, _, ok := runtime.Caller(0) + require.True(t, ok) + return filepath.Clean(filepath.Join(filepath.Dir(file), "../..")) +} + +func formalConformanceRegistryReadFile(t *testing.T, relativePath string) string { + t.Helper() + + content, err := os.ReadFile(filepath.Join(formalConformanceRegistryRepositoryRoot(t), relativePath)) + require.NoError(t, err) + return string(content) +} + +func formalConformanceRegistryBaselineRows(t *testing.T) []formalConformanceRegistryRow { + t.Helper() + + content := formalConformanceRegistryReadFile(t, "specs/awf-config-sources-compliance/README.md") + var rows []formalConformanceRegistryRow + for line := range strings.SplitSeq(content, "\n") { + if !strings.HasPrefix(line, "| T-DR-") { + continue + } + + cells := strings.Split(line, "|") + require.Len(t, cells, 6, "registry row: %s", line) + rows = append(rows, formalConformanceRegistryRow{ + TestID: strings.TrimSpace(cells[1]), + Requirement: strings.TrimSpace(cells[2]), + TestFile: strings.Trim(strings.TrimSpace(cells[4]), "`"), + }) + } + + require.NotEmpty(t, rows) + return rows +} + +func formalConformanceRegistryParseSeriesID(id string, prefix string) (int, bool) { + if !strings.HasPrefix(id, prefix) { + return 0, false + } + numeric := strings.TrimPrefix(id, prefix) + if len(numeric) < 3 { + return 0, false + } + for _, r := range numeric { + if r < '0' || r > '9' { + return 0, false + } + } + value, err := strconv.Atoi(numeric) + if err != nil { + return 0, false + } + return value, true +} + +func formalConformanceRegistryParsePlainID(id string) (int, bool) { + if strings.HasPrefix(id, "T-DR-SAFE-") { + return 0, false + } + return formalConformanceRegistryParseSeriesID(id, "T-DR-") +} + +func formalConformanceRegistryIsWellFormedFinalID(id string) bool { + if strings.HasPrefix(id, "T-DR-SAFE-") { + _, ok := formalConformanceRegistryParseSeriesID(id, "T-DR-SAFE-") + return ok + } + _, ok := formalConformanceRegistryParsePlainID(id) + return ok +} + +func formalConformanceRegistryNextPlainID(rows []formalConformanceRegistryRow) string { + max := 0 + for _, row := range rows { + value, ok := formalConformanceRegistryParsePlainID(row.TestID) + if !ok { + continue + } + if value > max { + max = value + } + } + return fmt.Sprintf("T-DR-%03d", max+1) +} + +func formalConformanceRegistryHasUniqueIDs(rows []formalConformanceRegistryRow) bool { + seen := make(map[string]struct{}, len(rows)) + for _, row := range rows { + if _, exists := seen[row.TestID]; exists { + return false + } + seen[row.TestID] = struct{}{} + } + return true +} + +func formalConformanceRegistryHasRequirementReference(row formalConformanceRegistryRow) bool { + return strings.TrimSpace(row.Requirement) != "" && strings.Contains(row.Requirement, "§") +} + +func formalConformanceRegistryHasImplementationFile(row formalConformanceRegistryRow) bool { + return strings.HasPrefix(row.TestFile, "pkg/workflow/") && strings.HasSuffix(row.TestFile, "_test.go") +} + +func formalConformanceRegistryRouteTestFile(spansDriftOutputAndSchema bool) string { + if spansDriftOutputAndSchema { + return "pkg/workflow/awf_config_drift_test.go" + } + return "pkg/workflow/awf_config_safeguards_formal_test.go" +} + +func formalConformanceRegistryHasSpecCrossReference(specContent, id string) bool { + for offset := 0; ; { + index := strings.Index(specContent[offset:], id) + if index < 0 { + return false + } + index += offset + end := index + len(id) + if (index == 0 || !formalConformanceRegistryIDCharacter(specContent[index-1])) && + (end == len(specContent) || !formalConformanceRegistryIDCharacter(specContent[end])) { + return true + } + offset = end + } +} + +func formalConformanceRegistryIDCharacter(character byte) bool { + return character >= 'A' && character <= 'Z' || + character >= 'a' && character <= 'z' || + character >= '0' && character <= '9' || + character == '-' +} + +func formalConformanceRegistrySeriesDisjoint(id string) bool { + plain := strings.HasPrefix(id, "T-DR-") && !strings.HasPrefix(id, "T-DR-SAFE-") + safe := strings.HasPrefix(id, "T-DR-SAFE-") + return plain != safe +} + +func TestFormalConformanceRegistry_P1_TestIDMonotonicity(t *testing.T) { + next := formalConformanceRegistryNextPlainID(formalConformanceRegistryBaselineRows(t)) + assert.Equal(t, "T-DR-011", next) + + nextValue, ok := formalConformanceRegistryParsePlainID(next) + require.True(t, ok) + assert.Equal(t, 11, nextValue) +} + +func TestFormalConformanceRegistry_P1_EmptyRegistryStartsAtOne(t *testing.T) { + assert.Equal(t, "T-DR-001", formalConformanceRegistryNextPlainID(nil)) +} + +func TestFormalConformanceRegistry_P2_TestIDNoDuplicates(t *testing.T) { + rows := formalConformanceRegistryBaselineRows(t) + assert.True(t, formalConformanceRegistryHasUniqueIDs(rows)) + + rows = append(rows, formalConformanceRegistryRow{TestID: "T-DR-010", Requirement: "§x", TestFile: "pkg/workflow/awf_config_drift_test.go"}) + assert.False(t, formalConformanceRegistryHasUniqueIDs(rows)) +} + +func TestFormalConformanceRegistry_P3_TestIDFormatWellFormed(t *testing.T) { + valid := []string{"T-DR-001", "T-DR-010", "T-DR-1000", "T-DR-SAFE-001", "T-DR-SAFE-1234"} + invalid := []string{"t-dr-001", "T-DR-01", "T-DR-ABC", "T-DRSAFE-001", "T-DR-SAFE-1", "T-DR-SAFE-01", "T-DR-SAFE-ABC"} + + for _, id := range valid { + assert.True(t, formalConformanceRegistryIsWellFormedFinalID(id), id) + } + for _, id := range invalid { + assert.False(t, formalConformanceRegistryIsWellFormedFinalID(id), id) + } +} + +func TestFormalConformanceRegistry_P4_PlaceholderIDRejectedAsFinal(t *testing.T) { + assert.False(t, formalConformanceRegistryIsWellFormedFinalID("T-DR-NNN")) +} + +func TestFormalConformanceRegistry_P5_RowHasRequirementReference(t *testing.T) { + for _, row := range formalConformanceRegistryBaselineRows(t) { + assert.True(t, formalConformanceRegistryHasRequirementReference(row), row.TestID) + } + assert.False(t, formalConformanceRegistryHasRequirementReference(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "required fields", TestFile: "pkg/workflow/awf_config_drift_test.go"})) +} + +func TestFormalConformanceRegistry_P6_RowHasImplementationFile(t *testing.T) { + for _, row := range formalConformanceRegistryBaselineRows(t) { + assert.True(t, formalConformanceRegistryHasImplementationFile(row), row.TestID) + assert.FileExists(t, filepath.Join(formalConformanceRegistryRepositoryRoot(t), row.TestFile), row.TestID) + } + assert.False(t, formalConformanceRegistryHasImplementationFile(formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "§3.1", TestFile: ""})) +} + +func TestFormalConformanceRegistry_P7_SafeguardRowRoutingDecision(t *testing.T) { + assert.Equal(t, "pkg/workflow/awf_config_safeguards_formal_test.go", formalConformanceRegistryRouteTestFile(false)) + assert.Equal(t, "pkg/workflow/awf_config_drift_test.go", formalConformanceRegistryRouteTestFile(true)) +} + +func TestFormalConformanceRegistry_P8_SpecCrossReferenceRequired(t *testing.T) { + specContent := formalConformanceRegistryReadFile(t, "specs/awf-config-sources-spec.md") + for _, row := range formalConformanceRegistryBaselineRows(t) { + assert.True(t, formalConformanceRegistryHasSpecCrossReference(specContent, row.TestID), row.TestID) + } + + assert.False(t, formalConformanceRegistryHasSpecCrossReference(specContent, "T-DR-011")) +} + +func TestFormalConformanceRegistry_P9_DriftSeriesVsSafeguardSeriesDisjoint(t *testing.T) { + assert.True(t, formalConformanceRegistrySeriesDisjoint("T-DR-010")) + assert.True(t, formalConformanceRegistrySeriesDisjoint("T-DR-SAFE-004")) + assert.False(t, formalConformanceRegistrySeriesDisjoint("T-DRX-010")) +} + +func TestFormalConformanceRegistry_EdgeCase_FourDigitRollover(t *testing.T) { + rows := []formalConformanceRegistryRow{{TestID: "T-DR-999", Requirement: "§x", TestFile: "pkg/workflow/awf_config_drift_test.go"}} + next := formalConformanceRegistryNextPlainID(rows) + assert.Equal(t, "T-DR-1000", next) + assert.True(t, formalConformanceRegistryIsWellFormedFinalID(next)) +} + +func TestFormalConformanceRegistry_EdgeCase_SafeguardOnlyRegistryDoesNotAffectPlainSeries(t *testing.T) { + rows := []formalConformanceRegistryRow{ + {TestID: "T-DR-SAFE-001", Requirement: "§8", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + {TestID: "T-DR-SAFE-004", Requirement: "§8", TestFile: "pkg/workflow/awf_config_safeguards_formal_test.go"}, + } + assert.Equal(t, "T-DR-001", formalConformanceRegistryNextPlainID(rows)) +} + +func TestFormalConformanceRegistry_EdgeCase_MissingImplementationFileIsInvalid(t *testing.T) { + row := formalConformanceRegistryRow{TestID: "T-DR-011", Requirement: "§3.1", TestFile: ""} + assert.False(t, formalConformanceRegistryHasImplementationFile(row)) +} diff --git a/specs/awf-config-sources-compliance/README.md b/specs/awf-config-sources-compliance/README.md index d8fd1ad43d9..0de78c3515f 100644 --- a/specs/awf-config-sources-compliance/README.md +++ b/specs/awf-config-sources-compliance/README.md @@ -12,18 +12,18 @@ defined in §3.1 of the specification for structured drift output. The following test IDs cover the `DriftRecord` schema and its usage requirements from §3.1 and §7.5. -| Test ID | Requirement | Description | -|---------|-------------|-------------| -| T-DR-001 | §3.1 — required fields | `DriftRecord` MUST include `property_path`, `drift_category`, `suggested_action`, and `detected_at`; records missing any required field are invalid and MUST be rejected. | -| T-DR-002 | §3.1 — `drift_category` enum | `drift_category` MUST be one of `missing_in_ghaw`, `missing_in_schema`, or `spec_mismatch`; any other value is invalid. | -| T-DR-003 | §3.1 — `detected_at` format | `detected_at` MUST be a valid ISO 8601 UTC timestamp; non-conforming values MUST be rejected. | -| T-DR-004 | §3.1 — `suggested_action` non-empty | `suggested_action` MUST NOT be empty (`minLength: 1`); an empty string MUST be rejected. | -| T-DR-005 | §3.1 — no additional properties | `DriftRecord` objects MUST NOT include properties beyond the four required fields; additional properties MUST be rejected. | -| T-DR-006 | §7.5.1 — corrective PR trigger | When any `DriftRecord` in the output list has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation MUST open a corrective PR (CR-05). | -| T-DR-007 | §7.5.1 — SLA escalation trigger | When CR-06 SLA window is exceeded and `DriftRecord` items with actionable categories are present, an escalation issue MUST be opened or updated. | -| T-DR-008 | §7.5.1 — corrective PR embeds records | The corrective PR description MUST embed the full `DriftRecord` list as JSON. | -| T-DR-009 | §7.5.1 — empty list is valid | An empty `DriftRecord` list (no drift detected) is a valid output and MUST NOT trigger corrective PR or escalation actions. | -| T-DR-010 | §7.2 Step 5 integration | The drift detection procedure Step 5 MUST produce a list of zero or more `DriftRecord` objects; the output format MUST be a JSON array conforming to the §3.1 schema. | +| Test ID | Requirement | Description | Implementation file | +|---------|-------------|-------------|---------------------| +| T-DR-001 | §3.1 — required fields | `DriftRecord` MUST include `property_path`, `drift_category`, `suggested_action`, and `detected_at`; records missing any required field are invalid and MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-002 | §3.1 — `drift_category` enum | `drift_category` MUST be one of `missing_in_ghaw`, `missing_in_schema`, or `spec_mismatch`; any other value is invalid. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-003 | §3.1 — `detected_at` format | `detected_at` MUST be a valid ISO 8601 UTC timestamp; non-conforming values MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-004 | §3.1 — `suggested_action` non-empty | `suggested_action` MUST NOT be empty (`minLength: 1`); an empty string MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-005 | §3.1 — no additional properties | `DriftRecord` objects MUST NOT include properties beyond the four required fields; additional properties MUST be rejected. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-006 | §7.5.1 — corrective PR trigger | When any `DriftRecord` in the output list has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation MUST open a corrective PR (CR-05). | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-007 | §7.5.1 — SLA escalation trigger | When CR-06 SLA window is exceeded and `DriftRecord` items with actionable categories are present, an escalation issue MUST be opened or updated. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-008 | §7.5.1 — corrective PR embeds records | The corrective PR description MUST embed the full `DriftRecord` list as JSON. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-009 | §7.5.1 — empty list is valid | An empty `DriftRecord` list (no drift detected) is a valid output and MUST NOT trigger corrective PR or escalation actions. | `pkg/workflow/awf_config_drift_test.go` | +| T-DR-010 | §7.2 Step 5 integration | The drift detection procedure Step 5 MUST produce a list of zero or more `DriftRecord` objects; the output format MUST be a JSON array conforming to the §3.1 schema. | `pkg/workflow/awf_config_drift_test.go` | --- @@ -31,12 +31,12 @@ The following test IDs cover the `DriftRecord` schema and its usage requirements The following test IDs cover the unavailable-source safeguards from §8. -| Test ID | Requirement | Description | -|---------|-------------|-------------| -| T-DR-SAFE-001 | §8 item 1 — snapshot storage and freshness | Every invocation MUST select the stable path for its runner type, expire snapshots older than 168 hours, mark expired-snapshot runs degraded, and SHOULD delete snapshots older than 14 days. | -| T-DR-SAFE-002 | §8 item 2 — retrieval warning | A canonical-source retrieval failure SHOULD identify the failing source paths and UTC timestamp. | -| T-DR-SAFE-003 | §8 item 3 — degraded-run safety | An unavailable or expired canonical source MUST mark the run degraded and MUST prevent destructive validation actions. | -| T-DR-SAFE-004 | §8 item 4 — scheduled persistence | A tracking issue SHOULD be opened or updated only when unavailability persists through the next scheduled cron invocation; manual and ad hoc runs do not advance the threshold. | +| Test ID | Requirement | Description | Implementation file | +|---------|-------------|-------------|---------------------| +| T-DR-SAFE-001 | §8 item 1 — snapshot storage and freshness | Every invocation MUST select the stable path for its runner type, expire snapshots older than 168 hours, mark expired-snapshot runs degraded, and SHOULD delete snapshots older than 14 days. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-002 | §8 item 2 — retrieval warning | A canonical-source retrieval failure SHOULD identify the failing source paths and UTC timestamp. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-003 | §8 item 3 — degraded-run safety | An unavailable or expired canonical source MUST mark the run degraded and MUST prevent destructive validation actions. | `pkg/workflow/awf_config_safeguards_formal_test.go` | +| T-DR-SAFE-004 | §8 item 4 — scheduled persistence | A tracking issue SHOULD be opened or updated only when unavailability persists through the next scheduled cron invocation; manual and ad hoc runs do not advance the threshold. | `pkg/workflow/awf_config_safeguards_formal_test.go` | --- diff --git a/specs/awf-config-sources-spec.md b/specs/awf-config-sources-spec.md index 9fd77a1c6f4..85fd5828cf0 100644 --- a/specs/awf-config-sources-spec.md +++ b/specs/awf-config-sources-spec.md @@ -110,6 +110,14 @@ A `DriftRecord` represents a single detected schema drift item. All automation a | `suggested_action` | `string` | **MUST** | Actionable remediation text; **MUST NOT** be empty | `pkg/workflow/awf_config_drift_formal_test.go` (`FormalDriftRecord.SuggestedAction`, `formalDriftRecordStructuralValidity`); production emission target: `pkg/workflow/` drift detection logic | | `detected_at` | `string` (ISO 8601) | **MUST** | UTC timestamp of detection; filesystem-safe format **SHOULD** use `YYYY-MM-DDTHH:MM:SSZ` | `pkg/workflow/awf_config_drift_formal_test.go` (`FormalDriftRecord.DetectedAt`); production emission target: `pkg/workflow/` drift detection logic | +The conformance fixture index assigns the following test IDs to the `DriftRecord` schema requirements: + +- Required fields: T-DR-001 +- `drift_category` enum: T-DR-002 +- `detected_at` format: T-DR-003 +- `suggested_action` non-empty: T-DR-004 +- No additional properties: T-DR-005 + ## 4. Required coverage checks When updating AWF config generation, schema sync, or validation in gh-aw, agents MUST verify: @@ -200,7 +208,7 @@ Drift detection MUST be triggered when: - **Missing in schema**: `gh-aw` generates a field not present in either schema. - **Spec mismatch**: CLI mapping in `gh-aw` disagrees with the normative spec description. -5. **Produce a drift report** listing: +5. **Produce a drift report** (T-DR-010) listing: - Each drifted property path (e.g., `apiProxy.anthropicAutoCache`). - Drift category (missing in gh-aw / missing in schema / spec mismatch). - Suggested corrective action (add coverage, open PR, update spec). @@ -279,7 +287,7 @@ A `DriftRecord` represents a single detected schema drift item produced by the d #### 7.5.1 Usage -The drift detection procedure (Section 7.2, Step 5) **MUST** produce a list of zero or more `DriftRecord` objects (schema: Section 3.1). When any record has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation **MUST** open a corrective PR (CR-05) and, if the SLA window is exceeded, an escalation issue (CR-06). The corrective PR description **MUST** embed the full `DriftRecord` list as JSON. +The drift detection procedure (Section 7.2, Step 5) **MUST** produce a list of zero or more `DriftRecord` objects (schema: Section 3.1; T-DR-009). When any record has `drift_category` of `missing_in_ghaw` or `spec_mismatch`, the detecting automation **MUST** open a corrective PR (CR-05; T-DR-006) and, if the SLA window is exceeded, an escalation issue (CR-06; T-DR-007). The corrective PR description **MUST** embed the full `DriftRecord` list as JSON (T-DR-008). **Example output (Step 5 of the drift detection procedure):**