Codex belt for #977

.agents/issue-977-ledger.yml

@@ -0,0 +1,160 @@
+version: 1
+issue: 977
+base: main
+branch: codex/issue-977
+tasks:
+  - id: task-01
+    title: Define alerting threshold schema in `config/alerting-thresholds.json`
+    status: done
+    started_at: '2026-01-20T06:17:28Z'
+    finished_at: '2026-01-20T06:17:37Z'
+    commit: fecc10aaeee9ba00d26ead35cb03111e5e51e188
+    notes: []
+  - id: task-02
+    title: Create `scripts/metrics_alerting.py` for threshold checks
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-03
+    title: Implement GitHub Issue creation for alerts in `scripts/metrics_alerting.py`
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-04
+    title: Add optional Slack webhook notification in `scripts/metrics_alerting.py`
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-05
+    title: Create scheduled workflow in `.github/workflows/metrics-alerting.yml` for
+      alerting checks
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-06
+    title: Document alerting configuration in `docs/agent-automation.md`
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-07
+    title: Define alerting threshold schema in config/
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-08
+    title: Create scripts/metrics_alerting.py for threshold checks
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-09
+    title: Implement GitHub Issue creation for alerts
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-10
+    title: Add optional Slack webhook notification
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-11
+    title: Create scheduled workflow for alerting checks
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-12
+    title: Document alerting configuration in docs/
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-13
+    title: 'Configurable thresholds for: success rate, duration, token usage'
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-14
+    title: Alerts create GitHub Issues with appropriate labels
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-15
+    title: Duplicate alerts are deduplicated (same issue not created twice)
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-16
+    title: Alert history tracked in metrics data
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-17
+    title: Documentation covers threshold tuning guidelines
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-18
+    title: 'Configurable thresholds for: success rate, duration, token usage'
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-19
+    title: Alerts create GitHub Issues with appropriate labels
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-20
+    title: Duplicate alerts are deduplicated (same issue not created twice)
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-21
+    title: Alert history tracked in metrics data
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []
+  - id: task-22
+    title: Documentation covers threshold tuning guidelines
+    status: todo
+    started_at: null
+    finished_at: null
+    commit: ''
+    notes: []

config/alerting-thresholds.json

@@ -0,0 +1,29 @@
+{
+  "schema_version": 1,
+  "updated": "2025-01-01",
+  "thresholds": {
+    "success_rate": {
+      "warning": 0.95,
+      "critical": 0.9,
+      "window_days": 7,
+      "min_runs": 20
+    },
+    "duration_ms": {
+      "p95_warning": 600000,
+      "p95_critical": 900000,
+      "window_days": 7,
+      "min_runs": 20
+    },
+    "token_usage": {
+      "p95_warning": 120000,
+      "p95_critical": 160000,
+      "window_days": 7,
+      "min_runs": 20
+    }
+  },
+  "issue_labels": [
+    "metrics",
+    "alert"
+  ],
+  "notes": "Thresholds apply to rolling windows for alerting checks."
+}

docs/agent-automation.md

@@ -158,6 +158,21 @@ After each major step, the workflow re-dispatches itself to continue the pipelin
 3. Review the run summary for readiness tables, watchdog escalation indicators, and Codex bootstrap status.
 4. Repeat manual dispatches as needed; scheduled runs provide 20-minute coverage for stale bootstrap detection.
 
+## Alerting Threshold Tuning
+
+Metrics alerting thresholds live in `config/alerting-thresholds.json` and should be tuned with both baseline behavior and
+alert fatigue in mind.
+
+- Start from a recent 30 to 60 day baseline for success rate, duration, and token usage. Use medians or p95s rather than
+  single-day outliers.
+- Prefer incremental adjustments (5 to 10 percent) instead of large jumps so you can attribute changes to real shifts.
+- Separate short-lived spikes from sustained drift. For example, only lower success rate thresholds after several runs show
+  the same regression.
+- Revisit thresholds after major workflow or model changes. Each new model or large prompt update can shift token usage and
+  duration distributions.
+- Keep alert volume manageable: if the same threshold fires repeatedly without action, relax it or add a higher-severity
+  tier so only actionable alerts page maintainers.
+
 ## Security Considerations
 
 - All sensitive operations continue to rely on `SERVICE_BOT_PAT` when available. The workflows gracefully fall back to