From fecc10aaeee9ba00d26ead35cb03111e5e51e188 Mon Sep 17 00:00:00 2001 From: stranske-automation-bot Date: Tue, 20 Jan 2026 06:17:28 +0000 Subject: [PATCH 1/7] chore(ledger): start task task-01 for issue #977 --- .agents/issue-977-ledger.yml | 160 +++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 .agents/issue-977-ledger.yml diff --git a/.agents/issue-977-ledger.yml b/.agents/issue-977-ledger.yml new file mode 100644 index 000000000..44c21cdd4 --- /dev/null +++ b/.agents/issue-977-ledger.yml @@ -0,0 +1,160 @@ +version: 1 +issue: 977 +base: main +branch: codex/issue-977 +tasks: + - id: task-01 + title: Define alerting threshold schema in `config/alerting-thresholds.json` + status: doing + started_at: '2026-01-20T06:17:28Z' + finished_at: null + commit: '' + notes: [] + - id: task-02 + title: Create `scripts/metrics_alerting.py` for threshold checks + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-03 + title: Implement GitHub Issue creation for alerts in `scripts/metrics_alerting.py` + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-04 + title: Add optional Slack webhook notification in `scripts/metrics_alerting.py` + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-05 + title: Create scheduled workflow in `.github/workflows/metrics-alerting.yml` for + alerting checks + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-06 + title: Document alerting configuration in `docs/agent-automation.md` + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-07 + title: Define alerting threshold schema in config/ + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-08 + title: Create scripts/metrics_alerting.py for threshold checks + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-09 + title: Implement GitHub Issue creation for alerts + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-10 + title: Add optional Slack webhook notification + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-11 + title: Create scheduled workflow for alerting checks + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-12 + title: Document alerting configuration in docs/ + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-13 + title: 'Configurable thresholds for: success rate, duration, token usage' + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-14 + title: Alerts create GitHub Issues with appropriate labels + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-15 + title: Duplicate alerts are deduplicated (same issue not created twice) + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-16 + title: Alert history tracked in metrics data + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-17 + title: Documentation covers threshold tuning guidelines + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-18 + title: 'Configurable thresholds for: success rate, duration, token usage' + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-19 + title: Alerts create GitHub Issues with appropriate labels + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-20 + title: Duplicate alerts are deduplicated (same issue not created twice) + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-21 + title: Alert history tracked in metrics data + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] + - id: task-22 + title: Documentation covers threshold tuning guidelines + status: todo + started_at: null + finished_at: null + commit: '' + notes: [] From 5617704288bac58b88b01aac7d08dbdcaf0b7b7e Mon Sep 17 00:00:00 2001 From: stranske-automation-bot Date: Tue, 20 Jan 2026 06:17:37 +0000 Subject: [PATCH 2/7] chore(ledger): finish task task-01 for issue #977 --- .agents/issue-977-ledger.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/issue-977-ledger.yml b/.agents/issue-977-ledger.yml index 44c21cdd4..114a9ed82 100644 --- a/.agents/issue-977-ledger.yml +++ b/.agents/issue-977-ledger.yml @@ -5,10 +5,10 @@ branch: codex/issue-977 tasks: - id: task-01 title: Define alerting threshold schema in `config/alerting-thresholds.json` - status: doing + status: done started_at: '2026-01-20T06:17:28Z' - finished_at: null - commit: '' + finished_at: '2026-01-20T06:17:37Z' + commit: fecc10aaeee9ba00d26ead35cb03111e5e51e188 notes: [] - id: task-02 title: Create `scripts/metrics_alerting.py` for threshold checks From 906a2b6fbe8f8c68fa58384f2937efe2ee1b6bda Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 20 Jan 2026 06:20:04 +0000 Subject: [PATCH 3/7] Add alerting thresholds configuration --- config/alerting-thresholds.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 config/alerting-thresholds.json diff --git a/config/alerting-thresholds.json b/config/alerting-thresholds.json new file mode 100644 index 000000000..01f2771b7 --- /dev/null +++ b/config/alerting-thresholds.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "updated": "2025-01-01", + "thresholds": { + "success_rate": { + "warning": 0.95, + "critical": 0.9, + "window_days": 7, + "min_runs": 20 + }, + "duration_ms": { + "p95_warning": 600000, + "p95_critical": 900000, + "window_days": 7, + "min_runs": 20 + }, + "token_usage": { + "p95_warning": 120000, + "p95_critical": 160000, + "window_days": 7, + "min_runs": 20 + } + }, + "issue_labels": [ + "metrics", + "alert" + ], + "notes": "Thresholds apply to rolling windows for alerting checks." +} From bae98bd5f92b4fb35f9629bcbe1bbbb8f3bae190 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 20 Jan 2026 06:31:21 +0000 Subject: [PATCH 4/7] Add metrics alerting checks and tests --- scripts/metrics_alerting.py | 602 +++++++++++++++++++++++++ tests/scripts/test_metrics_alerting.py | 297 ++++++++++++ 2 files changed, 899 insertions(+) create mode 100644 scripts/metrics_alerting.py create mode 100644 tests/scripts/test_metrics_alerting.py diff --git a/scripts/metrics_alerting.py b/scripts/metrics_alerting.py new file mode 100644 index 000000000..22bf40201 --- /dev/null +++ b/scripts/metrics_alerting.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +"""Evaluate metrics thresholds and open alert issues.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import requests + +from scripts import api_client, duplicate_detection +from src import percentile_calculator + +DEFAULT_METRICS_PATH = "metrics-history.ndjson" +DEFAULT_THRESHOLDS_PATH = "config/alerting-thresholds.json" +DEFAULT_ALERT_HISTORY_PATH = "metrics-history.ndjson" +DEFAULT_DEDUP_PAGES = 2 + + +@dataclass(frozen=True) +class Alert: + metric: str + severity: str + value: float + threshold: float + window_days: int + min_runs: int + sample_count: int + alert_key: str + details: dict[str, Any] + + +def _safe_float(value: Any) -> float | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _safe_int(value: Any) -> int | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _parse_timestamp(value: Any) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if not isinstance(value, str): + return None + raw = value.strip() + if not raw: + return None + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(raw) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed + + +def _read_ndjson(path: Path) -> tuple[list[dict[str, Any]], int]: + entries: list[dict[str, Any]] = [] + errors = 0 + try: + content = path.read_text(encoding="utf-8") + except OSError: + return entries, 1 + for line in content.splitlines(): + raw = line.strip() + if not raw: + continue + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + errors += 1 + continue + if isinstance(parsed, dict): + entries.append(parsed) + else: + errors += 1 + return entries, errors + + +def _filter_recent_entries( + entries: list[dict[str, Any]], + *, + window_days: int, + now: datetime, +) -> list[dict[str, Any]]: + cutoff = now - timedelta(days=window_days) + recent: list[dict[str, Any]] = [] + for entry in entries: + timestamp = _parse_timestamp(entry.get("timestamp")) + if timestamp is None: + continue + if timestamp >= cutoff: + recent.append(entry) + return recent + + +def _extract_success_rate(entry: dict[str, Any]) -> tuple[float | None, int | None, int | None]: + summary = entry.get("summary") + if isinstance(summary, dict): + passed = _safe_int(summary.get("passed")) + tests = _safe_int(summary.get("tests")) + if passed is not None and tests is not None and tests > 0: + return passed / tests, passed, tests + successes = _safe_int(summary.get("successes")) + total = _safe_int(summary.get("total")) + if successes is not None and total is not None and total > 0: + return successes / total, successes, total + rate = _safe_float(summary.get("success_rate")) + if rate is not None: + if rate > 1: + rate = rate / 100 + return rate, None, None + + rate = _safe_float(entry.get("success_rate")) + if rate is not None: + if rate > 1: + rate = rate / 100 + return rate, None, None + successes = _safe_int(entry.get("successes")) + total = _safe_int(entry.get("total")) + if successes is not None and total is not None and total > 0: + return successes / total, successes, total + return None, None, None + + +def _extract_duration_ms(entry: dict[str, Any]) -> float | None: + duration_ms = _safe_float(entry.get("duration_ms")) + if duration_ms is not None: + return duration_ms + summary = entry.get("summary") + if isinstance(summary, dict): + summary_ms = _safe_float(summary.get("duration_ms")) + if summary_ms is not None: + return summary_ms + summary_seconds = _safe_float(summary.get("duration_seconds")) + if summary_seconds is not None: + return summary_seconds * 1000 + duration_seconds = _safe_float(entry.get("duration_seconds")) + if duration_seconds is not None: + return duration_seconds * 1000 + return None + + +def _extract_token_usage(entry: dict[str, Any]) -> float | None: + token_usage = entry.get("token_usage") + if token_usage is None and isinstance(entry.get("summary"), dict): + token_usage = entry["summary"].get("token_usage") + if token_usage is None: + token_usage = entry.get("token_usage_total") + if isinstance(token_usage, dict): + total_tokens = _safe_float(token_usage.get("total_tokens")) + if total_tokens is not None: + return total_tokens + keys = ( + "input_tokens", + "output_tokens", + "reasoning_tokens", + "prompt_tokens", + "completion_tokens", + ) + values = [_safe_float(token_usage.get(key)) for key in keys] + numbers = [value for value in values if value is not None] + if numbers: + return sum(numbers) + return None + return _safe_float(token_usage) + + +def _compute_success_rate( + entries: list[dict[str, Any]], +) -> tuple[float | None, int, dict[str, Any]]: + total_successes = 0 + total_runs = 0 + rates: list[float] = [] + sample_count = 0 + + for entry in entries: + rate, successes, total = _extract_success_rate(entry) + if rate is None: + continue + sample_count += 1 + if successes is not None and total is not None: + total_successes += successes + total_runs += total + else: + rates.append(rate) + + if total_runs > 0: + value = total_successes / total_runs + details = { + "aggregation": "weighted", + "successes": total_successes, + "total": total_runs, + } + return value, sample_count, details + if rates: + value = sum(rates) / len(rates) + details = { + "aggregation": "average", + "samples": len(rates), + } + return value, sample_count, details + return None, sample_count, {} + + +def _compute_percentile( + entries: list[dict[str, Any]], + *, + value_extractor: Any, + percentile_value: float, +) -> tuple[float | None, int]: + values: list[float] = [] + for entry in entries: + value = value_extractor(entry) + if value is None: + continue + values.append(float(value)) + if not values: + return None, 0 + values.sort() + return percentile_calculator.percentile(values, percentile_value), len(values) + + +def _evaluate_lower_is_bad( + metric: str, + value: float | None, + sample_count: int, + threshold: dict[str, Any], +) -> Alert | None: + if value is None or sample_count < int(threshold["min_runs"]): + return None + warning = float(threshold["warning"]) + critical = float(threshold["critical"]) + severity = None + chosen_threshold = None + if value <= critical: + severity = "critical" + chosen_threshold = critical + elif value < warning: + severity = "warning" + chosen_threshold = warning + if severity is None: + return None + alert_key = f"metrics_alert:{metric}" + return Alert( + metric=metric, + severity=severity, + value=value, + threshold=chosen_threshold, + window_days=int(threshold["window_days"]), + min_runs=int(threshold["min_runs"]), + sample_count=sample_count, + alert_key=alert_key, + details={"thresholds": {"warning": warning, "critical": critical}}, + ) + + +def _evaluate_higher_is_bad( + metric: str, + value: float | None, + sample_count: int, + threshold: dict[str, Any], + *, + value_label: str, +) -> Alert | None: + if value is None or sample_count < int(threshold["min_runs"]): + return None + warning = float(threshold["p95_warning"]) + critical = float(threshold["p95_critical"]) + severity = None + chosen_threshold = None + if value >= critical: + severity = "critical" + chosen_threshold = critical + elif value >= warning: + severity = "warning" + chosen_threshold = warning + if severity is None: + return None + alert_key = f"metrics_alert:{metric}" + return Alert( + metric=metric, + severity=severity, + value=value, + threshold=chosen_threshold, + window_days=int(threshold["window_days"]), + min_runs=int(threshold["min_runs"]), + sample_count=sample_count, + alert_key=alert_key, + details={"thresholds": {"p95_warning": warning, "p95_critical": critical}, "value_label": value_label}, + ) + + +def build_alerts( + entries: list[dict[str, Any]], + thresholds: dict[str, Any], + *, + now: datetime | None = None, +) -> list[Alert]: + now = now or datetime.now(UTC) + alerts: list[Alert] = [] + threshold_set = thresholds.get("thresholds", {}) + + success_config = threshold_set.get("success_rate", {}) + if success_config: + success_entries = _filter_recent_entries( + entries, window_days=int(success_config["window_days"]), now=now + ) + success_rate, sample_count, details = _compute_success_rate(success_entries) + alert = _evaluate_lower_is_bad( + "success_rate", + success_rate, + sample_count, + success_config, + ) + if alert: + alerts.append(Alert(**{**alert.__dict__, "details": {**alert.details, **details}})) + + duration_config = threshold_set.get("duration_ms", {}) + if duration_config: + duration_entries = _filter_recent_entries( + entries, window_days=int(duration_config["window_days"]), now=now + ) + p95_duration, sample_count = _compute_percentile( + duration_entries, + value_extractor=_extract_duration_ms, + percentile_value=95, + ) + alert = _evaluate_higher_is_bad( + "duration_ms", + p95_duration, + sample_count, + duration_config, + value_label="p95_duration_ms", + ) + if alert: + alerts.append(alert) + + token_config = threshold_set.get("token_usage", {}) + if token_config: + token_entries = _filter_recent_entries( + entries, window_days=int(token_config["window_days"]), now=now + ) + p95_tokens, sample_count = _compute_percentile( + token_entries, + value_extractor=_extract_token_usage, + percentile_value=95, + ) + alert = _evaluate_higher_is_bad( + "token_usage", + p95_tokens, + sample_count, + token_config, + value_label="p95_tokens", + ) + if alert: + alerts.append(alert) + + return alerts + + +def _format_success_rate(value: float) -> str: + return f"{value * 100:.2f}%" + + +def _format_number(value: float) -> str: + if value >= 1000: + return f"{value:,.0f}" + return f"{value:.2f}" + + +def _format_alert_value(alert: Alert) -> str: + if alert.metric == "success_rate": + return _format_success_rate(alert.value) + return _format_number(alert.value) + + +def _format_threshold(alert: Alert) -> str: + if alert.metric == "success_rate": + return _format_success_rate(alert.threshold) + return _format_number(alert.threshold) + + +def _issue_title(alert: Alert) -> str: + return f"Metrics alert: {alert.metric} {alert.severity}" + + +def _issue_body(alert: Alert, now: datetime) -> str: + details = alert.details + lines = [ + f"Alert key: `{alert.alert_key}`", + "", + f"Metric: `{alert.metric}`", + f"Severity: **{alert.severity.upper()}**", + f"Window: last {alert.window_days} day(s)", + f"Samples: {alert.sample_count} (min {alert.min_runs})", + f"Observed value: {_format_alert_value(alert)}", + f"Threshold: {_format_threshold(alert)}", + f"Detected at: {now.isoformat().replace('+00:00', 'Z')}", + "", + "Thresholds:", + "```json", + json.dumps(details.get("thresholds", {}), indent=2, sort_keys=True), + "```", + ] + if alert.metric == "success_rate": + if "successes" in details and "total" in details: + lines.append(f"Successes: {details['successes']} / {details['total']}") + if "aggregation" in details: + lines.append(f"Aggregation: {details['aggregation']}") + return "\n".join(lines) + + +def _find_existing_issue( + repo: str, + token: str, + alert: Alert, + labels: list[str], +) -> duplicate_detection.SourceIssue | None: + return duplicate_detection.find_source_issue( + repo, + token, + query=alert.alert_key, + labels=labels, + pages=DEFAULT_DEDUP_PAGES, + ) + + +def _append_alert_history( + path: Path, + alert: Alert, + *, + issue_url: str | None, + issue_number: int | None, + issue_status: str, + now: datetime, +) -> None: + record = { + "metric_type": "alert", + "timestamp": now.isoformat().replace("+00:00", "Z"), + "alert_key": alert.alert_key, + "metric": alert.metric, + "severity": alert.severity, + "value": alert.value, + "threshold": alert.threshold, + "window_days": alert.window_days, + "min_runs": alert.min_runs, + "sample_count": alert.sample_count, + "issue": { + "status": issue_status, + "number": issue_number, + "url": issue_url, + }, + } + payload = json.dumps(record, sort_keys=True) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(payload + "\n") + + +def _send_slack_notification(webhook: str, alerts: list[Alert], issue_urls: list[str]) -> None: + lines = ["Metrics alerting summary:"] + for alert, issue_url in zip(alerts, issue_urls): + issue_text = issue_url or "no issue" + lines.append(f"- {alert.metric} ({alert.severity}): {_format_alert_value(alert)} -> {issue_text}") + payload = {"text": "\n".join(lines)} + response = requests.post(webhook, json=payload, timeout=10) + if response.status_code >= 400: + raise RuntimeError(f"Slack webhook returned {response.status_code}: {response.text}") + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Evaluate metrics alerting thresholds.") + parser.add_argument("--metrics-path", default=DEFAULT_METRICS_PATH, help="NDJSON metrics path") + parser.add_argument( + "--thresholds-path", + default=DEFAULT_THRESHOLDS_PATH, + help="Alerting thresholds config path", + ) + parser.add_argument( + "--alert-history-path", + default=DEFAULT_ALERT_HISTORY_PATH, + help="NDJSON metrics history output path", + ) + parser.add_argument("--repo", help="GitHub repository (owner/name)") + parser.add_argument( + "--token-env", + default="GITHUB_TOKEN", + help="Environment variable containing GitHub token", + ) + parser.add_argument( + "--slack-webhook", + help="Optional Slack webhook URL (overrides SLACK_WEBHOOK_URL env var)", + ) + parser.add_argument("--dry-run", action="store_true", help="Skip issue creation") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + thresholds_path = Path(args.thresholds_path) + if not thresholds_path.is_file(): + print(f"metrics_alerting: thresholds file not found: {thresholds_path}", file=sys.stderr) + return 1 + thresholds = json.loads(thresholds_path.read_text(encoding="utf-8")) + metrics_path = Path(args.metrics_path) + if not metrics_path.exists(): + print(f"metrics_alerting: metrics file not found: {metrics_path}", file=sys.stderr) + return 1 + + entries, errors = _read_ndjson(metrics_path) + if errors: + print(f"metrics_alerting: parse errors found: {errors}", file=sys.stderr) + + now = datetime.now(UTC) + alerts = build_alerts(entries, thresholds, now=now) + if not alerts: + print("metrics_alerting: no alerts triggered.") + return 0 + + repo = args.repo or os.environ.get("GITHUB_REPOSITORY") + token = os.environ.get(args.token_env, "") + labels = thresholds.get("issue_labels", []) + if not isinstance(labels, list): + labels = [] + + if not repo: + print("metrics_alerting: repo not set (use --repo or GITHUB_REPOSITORY).", file=sys.stderr) + return 1 + + issue_urls: list[str] = [] + for alert in alerts: + issue_url = None + issue_number = None + issue_status = "skipped" + if token and not args.dry_run: + existing = _find_existing_issue(repo, token, alert, labels) + if existing: + issue_url = existing.url + issue_number = existing.number + issue_status = "existing" + else: + issue = api_client.create_issue( + repo, + token, + _issue_title(alert), + _issue_body(alert, now), + labels, + ) + issue_url = issue.get("html_url") + issue_number = issue.get("number") + issue_status = "created" + elif not args.dry_run: + print("metrics_alerting: GitHub token not available.", file=sys.stderr) + return 1 + + _append_alert_history( + Path(args.alert_history_path), + alert, + issue_url=issue_url, + issue_number=issue_number, + issue_status=issue_status, + now=now, + ) + issue_urls.append(issue_url or "") + + slack_webhook = args.slack_webhook or os.environ.get("SLACK_WEBHOOK_URL") + if slack_webhook: + _send_slack_notification(slack_webhook, alerts, issue_urls) + + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/scripts/test_metrics_alerting.py b/tests/scripts/test_metrics_alerting.py new file mode 100644 index 000000000..ce8a2d247 --- /dev/null +++ b/tests/scripts/test_metrics_alerting.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from scripts import metrics_alerting + + +def _build_thresholds() -> dict: + return { + "thresholds": { + "success_rate": { + "warning": 0.95, + "critical": 0.9, + "window_days": 7, + "min_runs": 2, + }, + "duration_ms": { + "p95_warning": 800, + "p95_critical": 900, + "window_days": 7, + "min_runs": 2, + }, + "token_usage": { + "p95_warning": 1000, + "p95_critical": 2000, + "window_days": 7, + "min_runs": 2, + }, + }, + "issue_labels": ["metrics", "alert"], + } + + +def test_build_alerts_success_rate_warning() -> None: + thresholds = _build_thresholds() + now = datetime(2026, 1, 1, tzinfo=UTC) + entries = [ + { + "timestamp": "2026-01-01T00:00:00Z", + "summary": {"passed": 92, "tests": 100}, + }, + { + "timestamp": "2026-01-01T01:00:00Z", + "summary": {"passed": 46, "tests": 50}, + }, + ] + + alerts = metrics_alerting.build_alerts(entries, thresholds, now=now) + + assert len(alerts) == 1 + alert = alerts[0] + assert alert.metric == "success_rate" + assert alert.severity == "warning" + assert alert.sample_count == 2 + + +def test_build_alerts_duration_p95_critical() -> None: + thresholds = _build_thresholds() + now = datetime(2026, 1, 1, tzinfo=UTC) + entries = [ + {"timestamp": "2026-01-01T00:00:00Z", "duration_ms": 500}, + {"timestamp": "2026-01-01T01:00:00Z", "duration_ms": 600}, + {"timestamp": "2026-01-01T02:00:00Z", "duration_ms": 1000}, + ] + + alerts = metrics_alerting.build_alerts(entries, thresholds, now=now) + + assert len(alerts) == 1 + alert = alerts[0] + assert alert.metric == "duration_ms" + assert alert.severity == "critical" + assert alert.sample_count == 3 + + +def test_append_alert_history(tmp_path: Path) -> None: + alert = metrics_alerting.Alert( + metric="token_usage", + severity="warning", + value=1234.0, + threshold=1000.0, + window_days=7, + min_runs=2, + sample_count=3, + alert_key="metrics_alert:token_usage", + details={}, + ) + history_path = tmp_path / "metrics-history.ndjson" + now = datetime(2026, 1, 1, tzinfo=UTC) + + metrics_alerting._append_alert_history( + history_path, + alert, + issue_url="https://example.com/issue/1", + issue_number=1, + issue_status="created", + now=now, + ) + + line = history_path.read_text(encoding="utf-8").strip() + record = json.loads(line) + assert record["metric_type"] == "alert" + assert record["alert_key"] == "metrics_alert:token_usage" + assert record["issue"]["status"] == "created" + + +def test_main_creates_issue_and_records_history( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + thresholds = _build_thresholds() + thresholds_path = tmp_path / "thresholds.json" + thresholds_path.write_text(json.dumps(thresholds), encoding="utf-8") + metrics_path = tmp_path / "metrics.ndjson" + now = datetime.now(UTC) + metrics_path.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 92, "tests": 100}, + } + ), + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 46, "tests": 50}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + history_path = tmp_path / "history.ndjson" + + created = {} + + def fake_create_issue(repo, token, title, body, labels): + created["repo"] = repo + created["token"] = token + created["title"] = title + created["body"] = body + created["labels"] = labels + return {"html_url": "https://example.com/issue/1", "number": 1} + + monkeypatch.setattr(metrics_alerting.api_client, "create_issue", fake_create_issue) + monkeypatch.setattr( + metrics_alerting.duplicate_detection, "find_source_issue", lambda *_args, **_kwargs: None + ) + monkeypatch.setenv("GITHUB_REPOSITORY", "octo/test") + monkeypatch.setenv("GITHUB_TOKEN", "token") + + exit_code = metrics_alerting.main( + [ + "--metrics-path", + str(metrics_path), + "--thresholds-path", + str(thresholds_path), + "--alert-history-path", + str(history_path), + ] + ) + + assert exit_code == 0 + assert created["labels"] == ["metrics", "alert"] + assert "Metrics alert:" in created["title"] + history_entries = history_path.read_text(encoding="utf-8").strip().splitlines() + assert history_entries + + +def test_main_dedup_skips_issue_create( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + thresholds = _build_thresholds() + thresholds_path = tmp_path / "thresholds.json" + thresholds_path.write_text(json.dumps(thresholds), encoding="utf-8") + metrics_path = tmp_path / "metrics.ndjson" + now = datetime.now(UTC) + metrics_path.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 92, "tests": 100}, + } + ), + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 46, "tests": 50}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + history_path = tmp_path / "history.ndjson" + + def fake_create_issue(*_args, **_kwargs): + raise AssertionError("create_issue should not be called when deduped") + + monkeypatch.setattr(metrics_alerting.api_client, "create_issue", fake_create_issue) + monkeypatch.setattr( + metrics_alerting.duplicate_detection, + "find_source_issue", + lambda *_args, **_kwargs: metrics_alerting.duplicate_detection.SourceIssue( + number=1, + title="Metrics alert: success_rate warning", + body=None, + url="https://example.com/issue/1", + ), + ) + monkeypatch.setenv("GITHUB_REPOSITORY", "octo/test") + monkeypatch.setenv("GITHUB_TOKEN", "token") + + exit_code = metrics_alerting.main( + [ + "--metrics-path", + str(metrics_path), + "--thresholds-path", + str(thresholds_path), + "--alert-history-path", + str(history_path), + ] + ) + + assert exit_code == 0 + history_entries = history_path.read_text(encoding="utf-8").strip().splitlines() + assert history_entries + + +def test_main_sends_slack_notification( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + thresholds = _build_thresholds() + thresholds_path = tmp_path / "thresholds.json" + thresholds_path.write_text(json.dumps(thresholds), encoding="utf-8") + metrics_path = tmp_path / "metrics.ndjson" + now = datetime.now(UTC) + metrics_path.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 92, "tests": 100}, + } + ), + json.dumps( + { + "timestamp": now.isoformat().replace("+00:00", "Z"), + "summary": {"passed": 46, "tests": 50}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + history_path = tmp_path / "history.ndjson" + + monkeypatch.setattr(metrics_alerting.api_client, "create_issue", lambda *_: {}) + monkeypatch.setattr( + metrics_alerting.duplicate_detection, "find_source_issue", lambda *_args, **_kwargs: None + ) + monkeypatch.setenv("GITHUB_REPOSITORY", "octo/test") + monkeypatch.setenv("GITHUB_TOKEN", "token") + + called = {} + + def fake_slack(webhook, alerts, issue_urls): + called["webhook"] = webhook + called["count"] = len(alerts) + called["urls"] = issue_urls + + monkeypatch.setattr(metrics_alerting, "_send_slack_notification", fake_slack) + monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://example.com/hook") + + exit_code = metrics_alerting.main( + [ + "--metrics-path", + str(metrics_path), + "--thresholds-path", + str(thresholds_path), + "--alert-history-path", + str(history_path), + ] + ) + + assert exit_code == 0 + assert called["webhook"] == "https://example.com/hook" From 5c457ab1b82cead97359852027329946132afd20 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 20 Jan 2026 06:32:24 +0000 Subject: [PATCH 5/7] chore(autofix): formatting/lint --- scripts/metrics_alerting.py | 9 +++++++-- tests/scripts/test_metrics_alerting.py | 8 ++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/metrics_alerting.py b/scripts/metrics_alerting.py index 22bf40201..b4bad7957 100644 --- a/scripts/metrics_alerting.py +++ b/scripts/metrics_alerting.py @@ -312,7 +312,10 @@ def _evaluate_higher_is_bad( min_runs=int(threshold["min_runs"]), sample_count=sample_count, alert_key=alert_key, - details={"thresholds": {"p95_warning": warning, "p95_critical": critical}, "value_label": value_label}, + details={ + "thresholds": {"p95_warning": warning, "p95_critical": critical}, + "value_label": value_label, + }, ) @@ -487,7 +490,9 @@ def _send_slack_notification(webhook: str, alerts: list[Alert], issue_urls: list lines = ["Metrics alerting summary:"] for alert, issue_url in zip(alerts, issue_urls): issue_text = issue_url or "no issue" - lines.append(f"- {alert.metric} ({alert.severity}): {_format_alert_value(alert)} -> {issue_text}") + lines.append( + f"- {alert.metric} ({alert.severity}): {_format_alert_value(alert)} -> {issue_text}" + ) payload = {"text": "\n".join(lines)} response = requests.post(webhook, json=payload, timeout=10) if response.status_code >= 400: diff --git a/tests/scripts/test_metrics_alerting.py b/tests/scripts/test_metrics_alerting.py index ce8a2d247..e538f51ac 100644 --- a/tests/scripts/test_metrics_alerting.py +++ b/tests/scripts/test_metrics_alerting.py @@ -172,9 +172,7 @@ def fake_create_issue(repo, token, title, body, labels): assert history_entries -def test_main_dedup_skips_issue_create( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_main_dedup_skips_issue_create(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: thresholds = _build_thresholds() thresholds_path = tmp_path / "thresholds.json" thresholds_path.write_text(json.dumps(thresholds), encoding="utf-8") @@ -235,9 +233,7 @@ def fake_create_issue(*_args, **_kwargs): assert history_entries -def test_main_sends_slack_notification( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_main_sends_slack_notification(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: thresholds = _build_thresholds() thresholds_path = tmp_path / "thresholds.json" thresholds_path.write_text(json.dumps(thresholds), encoding="utf-8") From bbfef02b743aa82c6f161de1502e1e8c8b07fb6f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Jan 2026 06:34:52 +0000 Subject: [PATCH 6/7] chore(codex-keepalive): apply updates (PR #978) --- docs/agent-automation.md | 15 +++++++++++++++ scripts/metrics_alerting.py | 1 + 2 files changed, 16 insertions(+) diff --git a/docs/agent-automation.md b/docs/agent-automation.md index 2cbaa7506..ea3dcc0b3 100644 --- a/docs/agent-automation.md +++ b/docs/agent-automation.md @@ -158,6 +158,21 @@ After each major step, the workflow re-dispatches itself to continue the pipelin 3. Review the run summary for readiness tables, watchdog escalation indicators, and Codex bootstrap status. 4. Repeat manual dispatches as needed; scheduled runs provide 20-minute coverage for stale bootstrap detection. +## Alerting Threshold Tuning + +Metrics alerting thresholds live in `config/alerting-thresholds.json` and should be tuned with both baseline behavior and +alert fatigue in mind. + +- Start from a recent 30 to 60 day baseline for success rate, duration, and token usage. Use medians or p95s rather than + single-day outliers. +- Prefer incremental adjustments (5 to 10 percent) instead of large jumps so you can attribute changes to real shifts. +- Separate short-lived spikes from sustained drift. For example, only lower success rate thresholds after several runs show + the same regression. +- Revisit thresholds after major workflow or model changes. Each new model or large prompt update can shift token usage and + duration distributions. +- Keep alert volume manageable: if the same threshold fires repeatedly without action, relax it or add a higher-severity + tier so only actionable alerts page maintainers. + ## Security Considerations - All sensitive operations continue to rely on `SERVICE_BOT_PAT` when available. The workflows gracefully fall back to diff --git a/scripts/metrics_alerting.py b/scripts/metrics_alerting.py index b4bad7957..56ddcf4e4 100644 --- a/scripts/metrics_alerting.py +++ b/scripts/metrics_alerting.py @@ -425,6 +425,7 @@ def _issue_body(alert: Alert, now: datetime) -> str: f"Observed value: {_format_alert_value(alert)}", f"Threshold: {_format_threshold(alert)}", f"Detected at: {now.isoformat().replace('+00:00', 'Z')}", + "Tuning guide: docs/agent-automation.md#alerting-threshold-tuning", "", "Thresholds:", "```json", From e1f096b5419fef930d1371710443ddc3ec1bd919 Mon Sep 17 00:00:00 2001 From: stranske Date: Tue, 20 Jan 2026 06:56:10 +0000 Subject: [PATCH 7/7] fix: add explicit zip strict flag --- scripts/metrics_alerting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 scripts/metrics_alerting.py diff --git a/scripts/metrics_alerting.py b/scripts/metrics_alerting.py old mode 100644 new mode 100755 index 56ddcf4e4..573850dbc --- a/scripts/metrics_alerting.py +++ b/scripts/metrics_alerting.py @@ -489,7 +489,7 @@ def _append_alert_history( def _send_slack_notification(webhook: str, alerts: list[Alert], issue_urls: list[str]) -> None: lines = ["Metrics alerting summary:"] - for alert, issue_url in zip(alerts, issue_urls): + for alert, issue_url in zip(alerts, issue_urls, strict=False): issue_text = issue_url or "no issue" lines.append( f"- {alert.metric} ({alert.severity}): {_format_alert_value(alert)} -> {issue_text}"