From 4aeb0bba7f00ba37dff3d2af4b455e24f33d66e3 Mon Sep 17 00:00:00 2001 From: randypanding Date: Sat, 22 Aug 2026 03:57:37 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(dashboard):=20=E5=BA=A6=E9=87=8F=20pol?= =?UTF-8?q?icy=20=E9=98=88=E5=80=BC=E7=9C=9F=E6=BA=90=20metrics.yaml?= =?UTF-8?q?=EF=BC=88W5-C4=20.github#227=EF=BC=8CADR-0073=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 北极星护栏全集/注意力阈值/成本声明价/产品指标读取位/季度配额记录位, schema 自测锁完整性(缺护栏即红——互锁盲区=Goodhart 通道)。PR 1/5(堆叠基座)。Card: Cloudbird-Software/.github#227 --- governance/policy/metrics.yaml | 90 +++++++++++++++++ governance/tests/test-metrics-policy.sh | 123 ++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 governance/policy/metrics.yaml create mode 100644 governance/tests/test-metrics-policy.sh diff --git a/governance/policy/metrics.yaml b/governance/policy/metrics.yaml new file mode 100644 index 0000000..accf407 --- /dev/null +++ b/governance/policy/metrics.yaml @@ -0,0 +1,90 @@ +version: 1 +# ============================================================================ +# metrics.yaml —— 度量 dashboard 完整版阈值与声明位(W5-C4 .github#227,ADR-0073) +# 真源: 本文件(阈值/护栏/数据源声明唯一来源,宪法 §4A"阈值唯一来源"原则同源); +# 执行: governance/metrics.py(纯计算库,零网络)+ governance/dashboard-update.py +# (API 采集+呈现,butler-ledger 每 15min 驱动)+ governance/board-sync.py(板字段)。 +# 宪法: §8(度量全景——北极星是指标对非单一指标,防 Goodhart)/§12(状态可视化三投影) +# /§7(反退化——注意力成本如实入账不洗白)。 +# 原则: 数据源未落的指标一律声明 pending,dashboard 显示 pending 不显示 0 +# (ADR-0073 决策 7:缺数据不得被渲染成好数据)。 +# ============================================================================ + +north_star: + # 北极星对同屏互锁(AC-1 / ADR-0073 决策 1):零接触合并数 × 质量护栏同屏; + # 任一护栏 red → 合并数【显示归零】+原因标注(呈现层归零,原始计数保留在 JSON + # raw 字段——非数据删除);护栏回绿自动恢复显示。pending 护栏不触发归零 + # (缺证据≠劣化,按决策 7 独立显示盲区),但归零判定只认 red。 + window_days: 7 + guardrails: + # 逃逸率>0 持续:当前窗与上一窗(各 7 天,事件时戳直算——无跨轮状态残留)均>0 + # 分子=非演习 [auto-revert] PR + post-merge 冒烟 P0 issue(sli-report 同口径) + escape_rate_sustained: + red_when: "current_window_events>0 and previous_window_events>0" + # 回滚率:非演习 [auto-revert] PR 数 / 窗口内 merged PR 数(零分母=pending 不红) + revert_rate: + red_when_gt: 0.05 + # holdout 与主套件通过率差(ADR-0073 决策 1 三护栏之一;读取位未建——诚实 pending) + holdout_gap: + red_when_gt: 0.05 + data_source: pending + # 种子缺陷演习红率(应≈100%,ADR-0069);零可判定演习=pending 不红(不造 100%) + drill_red_rate: + red_when_lt: 1.0 + data_source: governance/drill/history.jsonl + # 仲裁误放行(arbiter 台账,ADR-0054 §7;窗口内 false-allow 行数>0 即 red) + false_allow: + red_when_gt: 0 + data_source: Cloudbird-Software/arbiter:tests/false_decision_ledger.jsonl + # 未经仲裁的状态变更泄漏(检测面未建——诚实 pending,不参与互锁触发) + state_change_leak: + red_when_gt: 0 + data_source: pending + +attention: + # 注意力会计(AC-2 / 宪法 §7):判断工作如实计量,防"签署退化成隐形 review" + # 签署耗时统计窗(type:intent issue 的 state:ir-draft→state:ir-signed timeline 差) + sign_window_days: 90 + # 签署耗时低于此秒数=可疑快速签署(退化信号计数——判断不可压缩,太快=没判断) + suspicious_fast_sign_seconds: 60 + # needs-human p90 停留超此小时数=整机停摆(宪法 §7:单独显示,不洗白成队列健康) + needs_human_p90_stop_hours: 24 + # 超时默认触发数 / 每合并 owner 分钟 / 周审计超时率:决策卡与审计包数据源未落 + # ——dashboard 显示 pending(不造数) + +security: + # 安全正确性(AC-2 / 宪法 §8):误放行/误拒窗口(台账按 date 字段过滤) + false_decision_window_days: 30 + # 误拒数:同台账 kind=false-deny(infra 不算误拒,ADR-0054 §7 约定) + # 陷阱拦截率(ADR-0071 W5-C2 硬谓词信任门)与泄漏数:数据源未落 → pending + +cost: + # 成本(AC-2 / 宪法 §8):单 IR 美元=(Actions 分钟+LLM token 归账)声明价折算/当月 IR 数 + # 全仓公开计费净额 $0(ADR-0020)——此为"失控速率"虚拟成本口径,非账单 + actions_price_per_minute_usd: 0.008 + llm_price_per_1k_tokens_usd: 0.002 + # 重数据源(CI-Workflows metering 归账,ADR-0062)快照复用 TTL:15min 刷新节奏下 + # 每 15min 拉 tarball 属浪费——快佐证龄>TTL 才重拉(快照与龄随 JSON 区回传) + snapshot_ttl_minutes: 60 + +user_results: + # 用户结果指标(AC-4 / 宪法 §8:防产品漂向"oracle 友好型") + # 各产品仓在 :metrics/user-result.yaml 声明一个用户侧结果指标: + # {metric_key: str, value: number, unit: str, updated_at: ISO}——文件缺失=pending + read_path: metrics/user-result.yaml + products: + - repo: Shorts_Director + - repo: Script_Writer + - repo: Use-up-Plan + - repo: AI_Web_School + - repo: mutual + # 季度难测产品/功能配额(配额制记录位——每季度刻意做一个难测的): + # owner 每季度在 entries 回填 {quarter: "2026-Q4", product: str, goal: str, status: planned|doing|done}; + # entries 空=本季未立(诚实显示,不预填占位值) + quarterly_hard_quota: + entries: [] + +board: + # factory-floor 板字段完善(AC-3 / 宪法 §12):谓词状态数据源=W5-C2 硬谓词信任门 + # (ADR-0071,进行中)——未落前板字段存在但值恒为 pending 标注(字段占位≠造数) + predicate_status_pending: "pending(W5-C2)" diff --git a/governance/tests/test-metrics-policy.sh b/governance/tests/test-metrics-policy.sh new file mode 100644 index 0000000..4eb8874 --- /dev/null +++ b/governance/tests/test-metrics-policy.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# test-metrics-policy.sh —— metrics.yaml policy schema 自测(W5-C4 .github#227,ADR-0073) +# +# 阈值唯一真源=policy 文件(宪法 §4A 同源原则):本测试锁定 schema 完整性, +# 缺护栏/缺阈值/畸形配额记录即红——防"互锁静默缺一路护栏"(北极星对漏一路 +# 等于 Goodhart 通道敞开)。数值语义(归零触发/聚合口径)由 test-metrics-northstar.sh +# 与 test-metrics-groups.sh 锁定。用法:bash governance/tests/test-metrics-policy.sh +# (零网络;需 python3+PyYAML——CI gate 预装,本地按 lib.sh 同款探测垫片) +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GOV="$(cd "$HERE/.." && pwd)" + +PASS=0; FAIL=0 +pass() { PASS=$((PASS+1)); echo "PASS $1"; } +fail() { FAIL=$((FAIL+1)); echo "FAIL $1"; } + +# --- python 解释器探测(CI 恒有 python3;本地 Git Bash python3 可能是商店 stub) --- +PY="" +for c in "${PYTHON:-}" python3 python py -3; do + [[ -n "$c" ]] || continue + "$c" -c 'import sys, yaml; print("ok")' >/dev/null 2>&1 || continue + PY="$c"; break +done +[[ -n "$PY" ]] || { echo "::error::无可用 python(含 pyyaml)"; exit 2; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT + +"$PY" - "$GOV/policy/metrics.yaml" "$GOV/REPOS.yaml" >"$TMP/out.txt" 2>"$TMP/err.txt" <<'PYEOF' +import sys, yaml + +errs = [] +def need(cond, msg): + if not cond: + errs.append(msg) + +try: + m = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) + repos = yaml.safe_load(open(sys.argv[2], encoding="utf-8")) +except Exception as e: + print(f"FATAL 解析失败: {e}", file=sys.stderr); sys.exit(2) + +need(isinstance(m, dict), "顶层非对象") +need(m.get("version") == 1, "version 须为 1") + +# --- 北极星护栏全集(缺一路=互锁盲区——ADR-0073 决策 1+卡面护栏面) --- +ns = m.get("north_star") or {} +GUARDS = ("escape_rate_sustained", "revert_rate", "holdout_gap", + "drill_red_rate", "false_allow", "state_change_leak") +g = ns.get("guardrails") or {} +for k in GUARDS: + need(k in g, f"north_star.guardrails 缺 {k}") +need(set(g) == set(GUARDS), f"guardrails 键集须恰为 {GUARDS}(多/少都算漂移),现={sorted(g)}") +need(isinstance(ns.get("window_days"), int) and ns["window_days"] > 0, + "north_star.window_days 须为正整数") +# 数据源 pending 必须显式声明(缺证据≠劣化,但盲区必须可见——ADR-0073 决策 7) +for k in ("holdout_gap", "state_change_leak"): + need((g.get(k) or {}).get("data_source") == "pending", + f"guardrails.{k}.data_source 须为 pending(读取位未建——诚实声明)") +need((g.get("drill_red_rate") or {}).get("data_source") == "governance/drill/history.jsonl", + "drill_red_rate 数据源须指向 drill 台账") +need("false_decision_ledger.jsonl" in str((g.get("false_allow") or {}).get("data_source")), + "false_allow 数据源须指向 arbiter 误放行台账") +# 数值阈值方向性(回滚率上限/演习红率下限) +rr = g.get("revert_rate") or {} +need(isinstance(rr.get("red_when_gt"), (int, float)) and 0 < rr["red_when_gt"] < 1, + "revert_rate.red_when_gt 须为 (0,1) 数值") +dr = g.get("drill_red_rate") or {} +need(isinstance(dr.get("red_when_lt"), (int, float)) and 0.5 <= dr["red_when_lt"] <= 1.0, + "drill_red_rate.red_when_lt 须为 [0.5,1.0](红率目标≈100%)") +fa = g.get("false_allow") or {} +need(fa.get("red_when_gt") == 0, "false_allow.red_when_gt 须为 0(一票即破线)") + +# --- 注意力会计阈值 --- +at = m.get("attention") or {} +need(isinstance(at.get("suspicious_fast_sign_seconds"), (int, float)) + and 0 < at["suspicious_fast_sign_seconds"] <= 600, + "suspicious_fast_sign_seconds 须为 (0,600] 秒(>10min 不再算「快速」签署)") +need(isinstance(at.get("needs_human_p90_stop_hours"), (int, float)) + and at["needs_human_p90_stop_hours"] == 24, + "needs_human_p90_stop_hours 须为 24(宪法 §7:超 1 天=整机停摆)") +need(isinstance(at.get("sign_window_days"), int) and at["sign_window_days"] > 0, + "sign_window_days 须为正整数") + +# --- 成本声明价 + 快照 TTL --- +co = m.get("cost") or {} +for k in ("actions_price_per_minute_usd", "llm_price_per_1k_tokens_usd"): + need(isinstance(co.get(k), (int, float)) and co[k] > 0, f"cost.{k} 须为正数") +need(isinstance(co.get("snapshot_ttl_minutes"), int) and co["snapshot_ttl_minutes"] >= 15, + "snapshot_ttl_minutes 须 ≥15(快于刷新节奏=每轮重拉,TTL 失效)") + +# --- 用户结果指标:产品清单非空、唯一、且都在组织地图内;配额记录位形状 --- +ur = m.get("user_results") or {} +prods = [p.get("repo") for p in (ur.get("products") or [])] +need(len(prods) >= 1, "user_results.products 须非空(宪法 §8:每产品至少一个指标位)") +need(len(prods) == len(set(prods)), f"products 有重复仓: {prods}") +declared = {r["name"] for r in (repos.get("repos") or [])} +for p in prods: + need(p in declared, f"产品仓 {p} 不在 REPOS.yaml 组织地图(GM-4 申报面)") +need(bool(ur.get("read_path")), "user_results.read_path 缺失(产品仓读取位约定)") +quota = (ur.get("quarterly_hard_quota") or {}).get("entries") or [] +need(isinstance(quota, list), "quarterly_hard_quota.entries 须为列表(记录位)") +for e in quota: + need(isinstance(e, dict) and e.get("quarter") and e.get("product"), + f"配额 entry 形状非法(须 quarter+product): {e}") + need(e.get("status") in (None, "planned", "doing", "done"), + f"配额 entry.status 非法: {e}") + +# --- 板字段 pending 标注(AC-3:谓词数据源未落=值 pending,不造数) --- +need(str((m.get("board") or {}).get("predicate_status_pending", "")).startswith("pending"), + "board.predicate_status_pending 须以 pending 开头") + +for e in errs: + print("ERR", e) +sys.exit(1 if errs else 0) +PYEOF +if [[ $? -eq 0 ]]; then + pass "metrics.yaml schema 完整(护栏全集/阈值方向/pending 声明/产品面/配额位)" +else + fail "metrics.yaml schema 校验失败:"; sed 's/^/ /' "$TMP/err.txt" "$TMP/out.txt" 2>/dev/null +fi + +echo "== test-metrics-policy: pass=$PASS fail=$FAIL ==" +[[ $FAIL -eq 0 ]] From 907ca4eb3bc7c36f482cc3e5968b3302e93719b3 Mon Sep 17 00:00:00 2001 From: randypanding Date: Sat, 22 Aug 2026 03:59:47 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat(dashboard):=20=E5=8C=97=E6=9E=81?= =?UTF-8?q?=E6=98=9F=E5=AF=B9=E4=BA=92=E9=94=81=E6=A0=B8=E5=BF=83=20metric?= =?UTF-8?q?s.py=EF=BC=88W5-C4=20.github#227=EF=BC=8CADR-0073=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 护栏三值判定(green/red/pending)+显示归零互锁(raw 保留非数据删除), fixture 自测 10 例(归零触发/单窗边界/零分母诚实/schema)。PR 2/5。Card: Cloudbird-Software/.github#227 --- governance/metrics.py | 130 ++++++++++++++++++++ governance/tests/test-metrics-northstar.sh | 136 +++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 governance/metrics.py create mode 100644 governance/tests/test-metrics-northstar.sh diff --git a/governance/metrics.py b/governance/metrics.py new file mode 100644 index 0000000..c76c1c9 --- /dev/null +++ b/governance/metrics.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""metrics.py —— 度量计算库(W5-C4 .github#227,ADR-0073;宪法 §8/§12) + +纯计算库:输入=采集层(dashboard-update.py / board-sync.py)拿到的原始数据结构, +输出=dashboard JSON 状态块 + human-brief 摘要。零网络、零时钟旁路(now 一律 +注入)——全部口径可离线 fixture 复算(owner 周审计独立复算权,宪法 §7)。 + +北极星对互锁(AC-1 / ADR-0073 决策 1,宪法级): + 零接触合并数 × 质量护栏是**一个指标对**。合并数单独上屏,Goodhart 定律保证 + 系统会牺牲质量刷合并数。互锁规则: + - 任一护栏 red → 合并数 display=0(呈现层显式归零 + 原因标注;raw 保留在 JSON + ——非数据删除,护栏回绿自动恢复显示) + - 护栏 pending(数据源未落/零分母)≠ 劣化,不触发归零,但独立显示盲区 + (决策 7:缺数据不得渲染成好数据,也不得渲染成坏数据) + - 演习红率目标 ≈100%(<阈值=关卡漏检=质量劣化,ADR-0069) + +阈值唯一来源 governance/policy/metrics.yaml(本库不内嵌任何阈值)。 +""" +import argparse +import json +import math +import os +import sys + +try: + import yaml +except ImportError: # pragma: no cover + print("FATAL 缺少 PyYAML(CI 预装;本地 pip install pyyaml)", file=sys.stderr) + raise SystemExit(2) + +DIR = os.path.dirname(os.path.abspath(__file__)) + +# 护栏渲染序(同屏呈现顺序=后果严重度:逃逸>回滚>演习>误放行>泄漏>通过率差) +GUARD_ORDER = ("escape_rate_sustained", "revert_rate", "drill_red_rate", + "false_allow", "state_change_leak", "holdout_gap") + + +def load_policy(path=None): + p = path or os.path.join(DIR, "policy", "metrics.yaml") + with open(p, encoding="utf-8") as f: + return yaml.safe_load(f) + + +def percentile(values, q): + """最近邻秩百分位(values 空→None:零分母诚实口径,不除零不出假值)。""" + if not values: + return None + s = sorted(values) + idx = max(0, min(len(s) - 1, math.ceil(q * len(s)) - 1)) + return s[idx] + + +def _guard_status(name, inp, policy): + """单护栏三值判定:green/red/pending(缺输入或零分母=pending,不造数)。""" + g = (policy.get("north_star") or {}).get("guardrails") or {} + if name == "escape_rate_sustained": + # 逃逸>0 持续=当前窗与上一窗均>0(事件时戳直算双窗,无跨轮状态残留) + if not isinstance(inp, dict) or "current" not in inp or "previous" not in inp: + return "pending", "数据源未接入" + cur, prev = inp["current"], inp["previous"] + val = {"current": cur, "previous": prev} + if cur > 0 and prev > 0: + return "red", f"逃逸持续:上一窗 {prev} + 本窗 {cur}([auto-revert]+post-merge P0)" + return "green", f"双窗逃逸 {prev}/{cur}" + if name == "revert_rate": + if not isinstance(inp, dict) or not inp.get("denom"): + return "pending", "零分母(窗口内无 merged PR——不除零,#98 T2)" + rate = inp["num"] / inp["denom"] + thr = g["revert_rate"]["red_when_gt"] + return ("red" if rate > thr else "green"), f"{inp['num']}/{inp['denom']}={rate:.3f}(阈 {thr})" + if name == "drill_red_rate": + if not isinstance(inp, dict) or not inp.get("denom"): + return "pending", "零可判定演习(红率不造 100%)" + rate = inp["red"] / inp["denom"] + thr = g["drill_red_rate"]["red_when_lt"] + return ("red" if rate < thr else "green"), f"红 {inp['red']}/{inp['denom']}={rate:.2f}(目标 ≈100%,阈 {thr})" + if name == "false_allow": + if inp is None: + return "pending", "arbiter 台账不可读(盲区独立显示,不冒充 0)" + return ("red" if inp > g["false_allow"]["red_when_gt"] else "green"), f"窗口内误放行 {inp} 例" + if name in ("state_change_leak", "holdout_gap"): + return "pending", f"数据源 pending({g[name].get('data_source')})——盲区独立显示" + return "pending", f"未知护栏 {name}" + + +def north_star(data, policy): + """北极星对同屏互锁(AC-1)。data 键:zero_touch_merges_7d:int|None + 各护栏输入。""" + raw = data.get("zero_touch_merges_7d") + guards, reasons = {}, [] + for name in GUARD_ORDER: + status, detail = _guard_status(name, data.get(name), policy) + guards[name] = {"status": status, "detail": detail} + if status == "red": + reasons.append(name) + zeroed = bool(reasons) # 呈现层归零=仅护栏 red;pending/零合并周不标注归零 + display = 0 if zeroed else (raw if raw is not None else 0) + return { + "zero_touch_merges_7d": { + "raw": raw, # 原始计数永不删除(归零只在呈现层) + "display": display, "zeroed": zeroed, + "zeroed_reasons": reasons, + "note": ("护栏破线期间的产出计数无意义——显示归零+原因标注;" + "raw 保留(ADR-0073 决策 1:呈现层归零,非数据删除)" + if zeroed else ("零接触合并周(分母为 0 的如实 0)" if raw == 0 else "护栏全绿——如实显示")), + }, + "guardrails": guards, + "interlocked_zeroed": zeroed, + "pending_blind_zones": [n for n in GUARD_ORDER if guards[n]["status"] == "pending"], + } + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + a = sub.add_parser("northstar", help="北极星互锁判定(fixture 输入→JSON 输出)") + a.add_argument("--input", required=True, help="JSON 文件:{zero_touch_merges_7d, escape_rate_sustained, ...}") + a.add_argument("--policy", default=None) + a.add_argument("--now", default=None, help="注入时钟(ISO)——离线复算用") + args = ap.parse_args(argv) + policy = load_policy(args.policy) + with open(args.input, encoding="utf-8") as f: + data = json.load(f) + if args.now: # 显式注入优先(owner 复算可复现;缺省取系统钟) + os.environ["METRICS_NOW"] = args.now + print(json.dumps(north_star(data, policy), ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/governance/tests/test-metrics-northstar.sh b/governance/tests/test-metrics-northstar.sh new file mode 100644 index 0000000..2f55de3 --- /dev/null +++ b/governance/tests/test-metrics-northstar.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# test-metrics-northstar.sh —— 北极星对互锁归零触发自测(W5-C4 AC-1,ADR-0073 决策 1) +# +# fixture 时间序列→护栏判定→显示层断言(零网络;实现=governance/metrics.py +# northstar 子命令——测试跑真实现不跑影子)。断言面: +# 互锁触发:任一护栏 red → display=0 + zeroed_reasons 标注 + raw 保留(非数据删除) +# 诚实 pending:零分母/数据源未落 → pending 不归零、不冒充 0 也不冒充绿 +# 恢复:护栏回绿 → display 恢复 raw +# 用法:bash governance/tests/test-metrics-northstar.sh +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GOV="$(cd "$HERE/.." && pwd)" + +PY="" +for c in "${PYTHON:-}" python3 python py -3; do + [[ -n "$c" ]] || continue + "$c" -c 'import sys, yaml; print("ok")' >/dev/null 2>&1 || continue + PY="$c"; break +done +[[ -n "$PY" ]] || { echo "::error::无可用 python(含 pyyaml)"; exit 2; } + +PASS=0; FAIL=0 +pass() { PASS=$((PASS+1)); echo "PASS $1"; } +fail() { FAIL=$((FAIL+1)); echo "FAIL $1"; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +M="$GOV/metrics.py" +run_case() { # run_case <名> <输入JSON> → $OUT(stdout JSON) + local name="$1" input="$2" + if ! OUT=$("$PY" "$M" northstar --input "$input" 2>"$TMP/err.txt"); then + fail "$name:metrics.py 退出非零($(cat "$TMP/err.txt"))"; OUT="{}"; return 1 + fi +} +jget() { # jget (eval 白名单=本测试内联表达式,无外部输入) + "$PY" -c "import json,sys;d=json.loads(sys.argv[1]);print(eval(sys.argv[2]))" "$1" "$2" 2>/dev/null | tr -d '\r' +} + +mkinput() { printf '%s' "$1" >"$TMP/in.json"; } + +# --- 基线:全护栏绿 → display=raw,未归零 --- +mkinput '{"zero_touch_merges_7d":12,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":12},"drill_red_rate":{"red":4,"denom":4},"false_allow":0}' +run_case "全绿" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 12 \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['zeroed']") == False \ + && $(jget "$OUT" "d['interlocked_zeroed']") == False ]] \ + && pass "全绿:display=12 未归零" || fail "全绿断言失败:$OUT" +} + +# --- 归零触发 1:逃逸持续(双窗>0)→ display=0 + 原因标注 + raw 保留 --- +mkinput '{"zero_touch_merges_7d":12,"escape_rate_sustained":{"current":2,"previous":1},"revert_rate":{"num":0,"denom":12},"drill_red_rate":{"red":4,"denom":4},"false_allow":0}' +run_case "逃逸持续" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['raw']") == 12 \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['zeroed']") == True \ + && $(jget "$OUT" "'escape_rate_sustained' in d['zero_touch_merges_7d']['zeroed_reasons']") == True \ + && $(jget "$OUT" "d['guardrails']['escape_rate_sustained']['status']") == red ]] \ + && pass "逃逸持续:显示归零+原因标注+raw=12 保留(呈现层归零非数据删除)" \ + || fail "逃逸持续断言失败:$OUT" +} + +# --- 逃逸单窗>0(未持续)不归零——持续条件是互锁触发边界 --- +mkinput '{"zero_touch_merges_7d":9,"escape_rate_sustained":{"current":1,"previous":0},"revert_rate":{"num":0,"denom":9},"drill_red_rate":{"red":4,"denom":4},"false_allow":0}' +run_case "逃逸单窗" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 9 \ + && $(jget "$OUT" "d['guardrails']['escape_rate_sustained']['status']") == green ]] \ + && pass "逃逸单窗(未持续):护栏绿不归零" || fail "逃逸单窗断言失败:$OUT" +} + +# --- 归零触发 2:演习红率破线(0.75<1.0——关卡漏检=质量劣化) --- +mkinput '{"zero_touch_merges_7d":12,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":12},"drill_red_rate":{"red":3,"denom":4},"false_allow":0}' +run_case "演习红率破线" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['zeroed_reasons'].count('drill_red_rate')") == 1 ]] \ + && pass "演习红率 0.75 破线:显示归零" || fail "演习红率断言失败:$OUT" +} + +# --- 归零触发 3:误放行 1 例即破线(一票否决) --- +mkinput '{"zero_touch_merges_7d":12,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":12},"drill_red_rate":{"red":4,"denom":4},"false_allow":1}' +run_case "误放行" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$OUT" "d['guardrails']['false_allow']['status']") == red ]] \ + && pass "误放行 1 例:显示归零(一票即破线)" || fail "误放行断言失败:$OUT" +} + +# --- 归零触发 4:回滚率超阈(2/12>5%) --- +mkinput '{"zero_touch_merges_7d":12,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":2,"denom":12},"drill_red_rate":{"red":4,"denom":4},"false_allow":0}' +run_case "回滚率超阈" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$OUT" "d['guardrails']['revert_rate']['status']") == red ]] \ + && pass "回滚率 0.167 超阈:显示归零" || fail "回滚率断言失败:$OUT" +} + +# --- 诚实 pending:零分母/数据源未落 → pending 不归零(缺数据≠劣化,决策 7) --- +mkinput '{"zero_touch_merges_7d":7,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":0},"drill_red_rate":{"red":0,"denom":0}}' +run_case "零分母周" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 7 \ + && $(jget "$OUT" "d['interlocked_zeroed']") == False \ + && $(jget "$OUT" "'revert_rate' in d['pending_blind_zones']") == True \ + && $(jget "$OUT" "'drill_red_rate' in d['pending_blind_zones']") == True \ + && $(jget "$OUT" "'holdout_gap' in d['pending_blind_zones']") == True ]] \ + && pass "零分母/未落数据源:pending 盲区可见但不归零" || fail "零分母断言失败:$OUT" +} + +# --- 误放行台账不可读 → pending 不冒充 0(fail-closed 呈现) --- +mkinput '{"zero_touch_merges_7d":5,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":5},"drill_red_rate":{"red":4,"denom":4}}' +run_case "台账不可读" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['guardrails']['false_allow']['status']") == pending \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 5 ]] \ + && pass "误放行数据缺失:pending(不冒充 0 也不触发归零)" || fail "台账缺失断言失败:$OUT" +} + +# --- 零合并周 + 护栏破线:display=0 且 zeroed=True(归零标注仍须给出原因) --- +mkinput '{"zero_touch_merges_7d":0,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":0},"drill_red_rate":{"red":1,"denom":4},"false_allow":0}' +run_case "零合并破线周" "$TMP/in.json" && { + [[ $(jget "$OUT" "d['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['zeroed']") == True \ + && $(jget "$OUT" "d['zero_touch_merges_7d']['raw']") == 0 ]] \ + && pass "零合并周破线:归零标注与原因照给(raw=0 如实)" || fail "零合并破线断言失败:$OUT" +} + +# --- JSON 输出 schema:键全集(agent 消费契约,schema v2 一部分) --- +mkinput '{"zero_touch_merges_7d":3,"escape_rate_sustained":{"current":0,"previous":0},"revert_rate":{"num":0,"denom":3},"drill_red_rate":{"red":4,"denom":4},"false_allow":0}' +run_case "schema" "$TMP/in.json" && { + "$PY" -c " +import json,sys +d=json.loads(sys.argv[1]) +ns=d['zero_touch_merges_7d'] +assert set(ns)=={'raw','display','zeroed','zeroed_reasons','note'}, ns.keys() +assert set(d)=={'zero_touch_merges_7d','guardrails','interlocked_zeroed','pending_blind_zones'}, d.keys() +assert set(d['guardrails'])=={'escape_rate_sustained','revert_rate','drill_red_rate','false_allow','state_change_leak','holdout_gap'} +assert all(set(g)=={'status','detail'} for g in d['guardrails'].values()) +" "$OUT" 2>/dev/null && pass "north_star JSON schema 键全集锁定" || fail "schema 断言失败:$OUT" +} + +echo "== test-metrics-northstar: pass=$PASS fail=$FAIL ==" +[[ $FAIL -eq 0 ]] From 25f9c179f95ae32360e0d49b6a4fff92d246a265 Mon Sep 17 00:00:00 2001 From: randypanding Date: Sat, 22 Aug 2026 04:02:58 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat(dashboard):=20=E5=9B=9B=E7=B1=BB?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E8=81=9A=E5=90=88+human-brief=20=E6=B8=B2?= =?UTF-8?q?=E6=9F=93=EF=BC=88W5-C4=20.github#227=EF=BC=8CADR-0073=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 注意力会计(签署 p50/p90/可疑快速签署/needs-human p90 停摆线)/安全正确性 (误放行窗过滤/演习分母口径)/成本(声明价折算/零 IR 不除零)/用户结果 (产品读取位 pending 不造数+季度配额记录位)+ eval CLI,fixture 自测 9 例。PR 3/5。Card: Cloudbird-Software/.github#227 --- governance/metrics.py | 187 +++++++++++++++++++++++- governance/tests/test-metrics-groups.sh | 155 ++++++++++++++++++++ 2 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 governance/tests/test-metrics-groups.sh diff --git a/governance/metrics.py b/governance/metrics.py index c76c1c9..35dad0b 100644 --- a/governance/metrics.py +++ b/governance/metrics.py @@ -17,6 +17,7 @@ 阈值唯一来源 governance/policy/metrics.yaml(本库不内嵌任何阈值)。 """ import argparse +import datetime as _dt import json import math import os @@ -109,20 +110,198 @@ def north_star(data, policy): } +# ---------- 四类指标(AC-2/AC-4,宪法 §8;缺数据=pending 不造数) ---------- + +def _fmt_s(sec): + """秒→人读时长(签署/停留呈现用)。""" + if sec is None: + return "pending" + if sec < 90: + return f"{sec:.0f}s" + if sec < 5400: + return f"{sec / 60:.1f}min" + return f"{sec / 3600:.1f}h" + + +def attention_stats(data, policy): + """注意力会计:签署耗时/needs-human p90/超时默认触发数/可疑快速签署数(宪法 §7)。""" + at = policy["attention"] + durations = data.get("sign_durations_seconds") or [] + dwell = data.get("needs_human_dwell_hours") or [] + p90 = percentile(dwell, 0.90) + stop_h = at["needs_human_p90_stop_hours"] + return { + "sign_count": len(durations), + "sign_p50_seconds": percentile(durations, 0.50), + "sign_p90_seconds": percentile(durations, 0.90), + "sign_in_flight": data.get("sign_in_flight", 0), # 仍 ir-draft 未签(不计耗时统计) + "suspicious_fast_signs": sum(1 for s in durations if s < at["suspicious_fast_sign_seconds"]), + "suspicious_fast_sign_seconds": at["suspicious_fast_sign_seconds"], + "needs_human_count": len(dwell), + "needs_human_p90_hours": p90, + "needs_human_p90_stop": (None if p90 is None else bool(p90 > stop_h)), + "needs_human_stop_hours": stop_h, + # 数据源未落(决策卡/审计包基建在后续波次)——pending 诚实显示,不冒充 0 + "timeout_defaults": "pending:决策卡超时默认触发数(数据源未落)", + "owner_minutes_per_merge": "pending:每合并 owner 分钟(评审事件流未落)", + "audit_overtime_rate": "pending:周审计超时率(审计包组装未落)", + } + + +def security_stats(data, policy): + """安全正确性:误放行/误拒(arbiter 台账窗内)、泄漏、演习红率、陷阱拦截率。""" + win_days = policy["security"]["false_decision_window_days"] + now = _parse_iso(data.get("now")) + allow = deny = 0 + for rec in data.get("false_decision_lines") or []: + ts = _parse_iso(rec.get("date")) + if ts and now and (now - ts).days <= win_days: + if rec.get("kind") == "false-allow": + allow += 1 + elif rec.get("kind") == "false-deny": + deny += 1 + drills = data.get("drill_records") or [] + reds = sum(1 for r in drills if r.get("verdict") == "red") + denom = sum(1 for r in drills if r.get("verdict") in ("red", "green")) # no-surface 不入分母 + return { + "false_allow_window": allow, + "false_deny_window": deny, + "false_decision_window_days": win_days, + "drill_red": reds, "drill_denom": denom, + "state_change_leaks": "pending:未经仲裁的状态变更泄漏检测面未建", + "trap_intercept_rate": "pending:陷阱拦截率(ADR-0071 W5-C2 信任门未落)", + } + + +def cost_stats(data, policy): + """成本:单 IR 美元(声明价折算——公开仓计费净额 $0,防失控速率的虚拟口径)。""" + co = policy["cost"] + minutes = data.get("actions_minutes_month") + tokens = data.get("llm_tokens_month") + irs = data.get("ir_count_month") + per_ir = None + if minutes is not None and tokens is not None and irs: + per_ir = round((minutes * co["actions_price_per_minute_usd"] + + tokens / 1000 * co["llm_price_per_1k_tokens_usd"]) / irs, 4) + return { + "actions_minutes_month": minutes, + "llm_tokens_month": tokens, + "ir_count_month": irs, + "per_ir_usd": per_ir, + "per_ir_usd_note": ("pending:当月零 IR——不除零(#98 T2)" if irs == 0 + else "声明价折算(actions $/min × 分钟 + LLM $/1k × token)/ 当月 IR 数"), + "actions_price_per_minute_usd": co["actions_price_per_minute_usd"], + "llm_price_per_1k_tokens_usd": co["llm_price_per_1k_tokens_usd"], + "snapshot_age_minutes": data.get("cost_snapshot_age_minutes"), + "butler_usd_week": "pending:管家美元/周(按 workflow 分钟拆账未落)", + "patrol_yield": "pending:patrol yield(W3-C2 demo-probe 期,真实 bug 计数为 0 分母)", + } + + +def user_results_stats(data, policy): + """用户结果指标:各产品仓声明读取位(文件缺失=pending)+ 季度难测配额记录位。""" + ur = policy["user_results"] + files = data.get("user_metric_files") or {} + products = {} + for p in ur.get("products") or []: + repo = p["repo"] + m = files.get(repo) + if isinstance(m, dict) and m.get("metric_key") and "value" in m: + products[repo] = {"status": "ok", **m} + else: + products[repo] = {"status": "pending", + "detail": f"产品仓未声明用户结果指标({ur['read_path']} 缺失——埋点滞后,ADR-0073 后果节)"} + quota = (ur.get("quarterly_hard_quota") or {}).get("entries") or [] + return {"products": products, + "quarterly_hard_quota": {"entries": quota, + "note": "配额制:每季度刻意做一个难测的——entries 空=本季未立(记录位,owner 回填)"}} + + +def build_payload(data, policy): + """schema v2 组装:north_star + metrics 四组(generated_at 由采集层注入)。""" + return { + "generated_at": data.get("generated_at"), + "north_star": north_star(data, policy), + "metrics": { + "attention": attention_stats(data, policy), + "security": security_stats(data, policy), + "cost": cost_stats(data, policy), + "user_results": user_results_stats(data, policy), + }, + } + + +# ---------- human-brief 渲染(宪法 §8"人 30 秒读懂";正文顶部=北极星对) ---------- + +def render_brief(payload): + ns, m = payload["north_star"], payload["metrics"] + z = ns["zero_touch_merges_7d"] + if z["zeroed"]: + head = (f"零接触合并数(近 7 天):**0(显示归零——护栏破线:" + f"{'、'.join(z['zeroed_reasons'])})**;原始计数 {z['raw']} 保留在 JSON raw" + "(呈现层归零,非数据删除)") + elif z["raw"] == 0: + head = "零接触合并数(近 7 天):**0**(零接触合并周——如实 0,非归零)" + else: + head = f"零接触合并数(近 7 天):**{z['raw']}**(护栏全绿——如实显示)" + gtxt = " · ".join(f"{n} {ns['guardrails'][n]['status']}" for n in GUARD_ORDER) + glines = "\n".join(f" - {n}: **{g['status']}**({g['detail']})" + for n, g in ns["guardrails"].items()) + at, se, co, ur = m["attention"], m["security"], m["cost"], m["user_results"] + p90h = "pending" if at["needs_human_p90_hours"] is None else f"{at['needs_human_p90_hours']:.0f}h" + per_ir = "pending" if co["per_ir_usd"] is None else f"${co['per_ir_usd']}" + prod_txt = " · ".join(f"{r}: {v['status']}" + (f"({v['metric_key']}={v['value']}{v.get('unit', '')})" if v["status"] == "ok" else "") + for r, v in sorted(ur["products"].items())) + quota = "、".join(f"{e['quarter']} {e['product']}({e.get('status', 'planned')})" + for e in ur["quarterly_hard_quota"]["entries"]) or "本季未立(记录位空——诚实显示)" + return f"""## 北极星对(同屏互锁 · 宪法 §8 / ADR-0073 决策 1) + +{head} +质量护栏:{gtxt} +{glines} +盲区(pending,不参与归零判定——缺数据≠劣化):{'、'.join(ns['pending_blind_zones']) or '无'} + +## 四类指标(宪法 §8 全景) + +- 注意力会计:签署 {at['sign_count']} 例(p50 {_fmt_s(at['sign_p50_seconds'])} / p90 {_fmt_s(at['sign_p90_seconds'])},在途 {at['sign_in_flight']})· 可疑快速签署(<{at['suspicious_fast_sign_seconds']}s){at['suspicious_fast_signs']} 例 · needs-human {at['needs_human_count']} 张 p90 停留 {p90h}(>{at['needs_human_stop_hours']}h=整机停摆线)· 超时默认触发 pending +- 安全正确性({se['false_decision_window_days']} 天窗):误放行 {se['false_allow_window']} · 误拒 {se['false_deny_window']} · 演习红率 {se['drill_red']}/{se['drill_denom']} · 泄漏 pending · 陷阱拦截率 pending +- 成本:单 IR {per_ir}(Actions {co['actions_minutes_month'] if co['actions_minutes_month'] is not None else 'pending'} 分钟 + LLM {co['llm_tokens_month'] if co['llm_tokens_month'] is not None else 'pending'} token,声明价)· 管家美元/周 pending · patrol yield pending +- 用户结果:{prod_txt} +- 季度难测配额:{quota} +""" + + +def _parse_iso(s): + if not s: + return None + try: + return _dt.datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except ValueError: + return None + + def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) a = sub.add_parser("northstar", help="北极星互锁判定(fixture 输入→JSON 输出)") a.add_argument("--input", required=True, help="JSON 文件:{zero_touch_merges_7d, escape_rate_sustained, ...}") a.add_argument("--policy", default=None) - a.add_argument("--now", default=None, help="注入时钟(ISO)——离线复算用") + a = sub.add_parser("eval", help="全量指标计算(fixture/采集层输入→payload JSON + human-brief)") + a.add_argument("--input", required=True, help="JSON 文件(dashboard 数据结构,见各 *_stats docstring)") + a.add_argument("--policy", default=None) + a.add_argument("--render", action="store_true", help="附加 human-brief markdown(分节符 ==== 后输出)") args = ap.parse_args(argv) policy = load_policy(args.policy) with open(args.input, encoding="utf-8") as f: data = json.load(f) - if args.now: # 显式注入优先(owner 复算可复现;缺省取系统钟) - os.environ["METRICS_NOW"] = args.now - print(json.dumps(north_star(data, policy), ensure_ascii=False, indent=2)) + if args.cmd == "northstar": + print(json.dumps(north_star(data, policy), ensure_ascii=False, indent=2)) + return 0 + payload = build_payload(data, policy) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + if args.render: + print("\n==== human-brief ====\n") + print(render_brief(payload), end="") return 0 diff --git a/governance/tests/test-metrics-groups.sh b/governance/tests/test-metrics-groups.sh new file mode 100644 index 0000000..32bf837 --- /dev/null +++ b/governance/tests/test-metrics-groups.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# test-metrics-groups.sh —— 四类指标聚合 + human-brief 渲染自测(W5-C4 AC-2/AC-4,ADR-0073) +# +# fixture 时间序列→p90/签署/率值/成本折算断言(零网络;实现=governance/metrics.py +# eval 子命令)。锁定口径: +# p90 最近邻秩(10 样本 1..10h → 9h)· 可疑快速签署按 policy 阈值计数 +# 误放行/误拒窗过滤(窗内外分离)· 演习 no-surface 不入分母 +# 成本声明价折算 + 零 IR 不除零 · 用户结果 pending 不造数 · 渲染含归零标注 +# 用法:bash governance/tests/test-metrics-groups.sh +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GOV="$(cd "$HERE/.." && pwd)" + +PY="" +for c in "${PYTHON:-}" python3 python py -3; do + [[ -n "$c" ]] || continue + "$c" -c 'import sys, yaml; print("ok")' >/dev/null 2>&1 || continue + PY="$c"; break +done +[[ -n "$PY" ]] || { echo "::error::无可用 python(含 pyyaml)"; exit 2; } + +PASS=0; FAIL=0 +pass() { PASS=$((PASS+1)); echo "PASS $1"; } +fail() { FAIL=$((FAIL+1)); echo "FAIL $1"; } +jget() { "$PY" -c "import json,sys;d=json.loads(sys.argv[1]);print(eval(sys.argv[2]))" "$1" "$2" 2>/dev/null | tr -d '\r'; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +M="$GOV/metrics.py" + +cat >"$TMP/in.json" <<'EOF' +{ + "now": "2026-08-22T00:00:00Z", + "generated_at": "2026-08-22T00:00:00Z", + "zero_touch_merges_7d": 8, + "escape_rate_sustained": {"current": 0, "previous": 0}, + "revert_rate": {"num": 0, "denom": 8}, + "drill_red_rate": {"red": 3, "denom": 3}, + "false_allow": 0, + "sign_durations_seconds": [30, 120, 3600, 7200], + "sign_in_flight": 2, + "needs_human_dwell_hours": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "false_decision_lines": [ + {"date": "2026-08-10T00:00:00Z", "kind": "false-allow"}, + {"date": "2026-08-15T00:00:00Z", "kind": "false-deny"}, + {"date": "2026-05-01T00:00:00Z", "kind": "false-allow"}, + {"date": "2026-08-20T00:00:00Z", "kind": "infra"} + ], + "drill_records": [ + {"verdict": "red"}, {"verdict": "red"}, {"verdict": "green"}, {"verdict": "no-surface"} + ], + "actions_minutes_month": 100, + "llm_tokens_month": 50000, + "ir_count_month": 5, + "user_metric_files": {"mutual": {"metric_key": "ndcg@5", "value": 0.41, "unit": "", "updated_at": "2026-08-21T00:00:00Z"}} +} +EOF + +if ! OUT=$("$PY" "$M" eval --input "$TMP/in.json" --render 2>"$TMP/err.txt"); then + echo "::error::metrics.py eval 失败:$(cat "$TMP/err.txt")"; exit 2 +fi +JSON="${OUT%%====*}" + +# --- 注意力会计:p50/p90 最近邻秩 + 可疑快速签署(阈值来自 policy=60s) --- +[[ $(jget "$JSON" "d['metrics']['attention']['sign_p50_seconds']") == 120 \ + && $(jget "$JSON" "d['metrics']['attention']['sign_p90_seconds']") == 7200 \ + && $(jget "$JSON" "d['metrics']['attention']['suspicious_fast_signs']") == 1 \ + && $(jget "$JSON" "d['metrics']['attention']['sign_in_flight']") == 2 ]] \ + && pass "签署 p50=120s/p90=7200s(最近邻秩),可疑快速签署 1 例(30s<60s 阈)" \ + || fail "签署统计断言失败" + +# --- needs-human p90:10 样本 1..10h → p90=9h(未破 24h 停摆线) --- +[[ $(jget "$JSON" "d['metrics']['attention']['needs_human_p90_hours']") == 9 \ + && $(jget "$JSON" "d['metrics']['attention']['needs_human_p90_stop']") == False ]] \ + && pass "needs-human p90=9h(最近邻秩),停摆线未破" \ + || fail "needs-human p90 断言失败" + +# --- 安全正确性:窗过滤(30 天窗:1 误放行+1 误拒;窗外与 infra 不计)+ 演习分母 --- +[[ $(jget "$JSON" "d['metrics']['security']['false_allow_window']") == 1 \ + && $(jget "$JSON" "d['metrics']['security']['false_deny_window']") == 1 \ + && $(jget "$JSON" "d['metrics']['security']['drill_red']") == 2 \ + && $(jget "$JSON" "d['metrics']['security']['drill_denom']") == 3 ]] \ + && pass "误放行/误拒窗内各 1(窗外与 infra 不计);演习 2 红/3 可判定(no-surface 不入分母)" \ + || fail "安全正确性断言失败" + +# --- 成本:声明价折算 (100*0.008 + 50*0.002)/5 = 0.18 美元/IR --- +[[ $(jget "$JSON" "d['metrics']['cost']['per_ir_usd']") == 0.18 ]] \ + && pass '单 IR $0.18 =(100min×$0.008 + 50k token×$0.002)/ 5 IR' \ + || fail "成本折算断言失败(期望 0.18)" + +# --- 用户结果:声明仓 ok、其余 pending;配额空=未立 --- +[[ $(jget "$JSON" "d['metrics']['user_results']['products']['mutual']['status']") == ok \ + && $(jget "$JSON" "d['metrics']['user_results']['products']['mutual']['value']") == 0.41 \ + && $(jget "$JSON" "d['metrics']['user_results']['products']['Shorts_Director']['status']") == pending \ + && $(jget "$JSON" "len(d['metrics']['user_results']['quarterly_hard_quota']['entries'])") == 0 ]] \ + && pass "用户结果:mutual ok(0.41)、未声明仓 pending、配额记录位空=未立" \ + || fail "用户结果断言失败" + +# --- 渲染(AC-1 同屏 + §8 人 30 秒):北极星对在最顶,归零时含标注与 raw 保留 --- +BRIEF="${OUT#*====}" +if grep -q "^## 北极星对" <<<"$OUT" && grep -q "零接触合并数(近 7 天):\*\*8\*\*" <<<"$BRIEF" \ + && grep -q "## 四类指标" <<<"$BRIEF" && grep -q "可疑快速签署(<60s)1 例" <<<"$BRIEF" \ + && grep -q "needs-human 10 张 p90 停留 9h" <<<"$BRIEF" && grep -q "误放行 1 · 误拒 1" <<<"$BRIEF" \ + && grep -q "单 IR \$0.18" <<<"$BRIEF" && grep -q "季度难测配额:本季未立" <<<"$BRIEF"; then + pass "human-brief:北极星对置顶+四类指标一行一类+30 秒可读" +else fail "human-brief 渲染断言失败"; fi + +# --- 渲染归零形态:护栏破线 → 顶部显示 0+原因+raw 保留(AC-1 呈现层断言) --- +# 路径经 argv 传递(MSYS 自动转换——嵌入代码字符串的 /tmp 路径原生 python 不识别) +"$PY" - "$TMP/in.json" "$TMP/red.json" <<'PYEOF' +import json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +d["drill_red_rate"] = {"red": 1, "denom": 3} # 0.33 < 1.0 破线 +d["drill_records"] = [{"verdict": "red"}, {"verdict": "green"}, {"verdict": "green"}] +json.dump(d, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False) +PYEOF +OUT2=$("$PY" "$M" eval --input "$TMP/red.json" --render 2>/dev/null) || { echo "::error::red eval 失败"; exit 2; } +JSON2="${OUT2%%====*}"; BRIEF2="${OUT2#*====}" +[[ $(jget "$JSON2" "d['north_star']['zero_touch_merges_7d']['display']") == 0 \ + && $(jget "$JSON2" "d['north_star']['zero_touch_merges_7d']['raw']") == 8 ]] \ + && grep -q "显示归零——护栏破线:drill_red_rate" <<<"$BRIEF2" \ + && grep -q "原始计数 8 保留" <<<"$BRIEF2" \ + && pass "渲染归零形态:0+破线路由标注+raw=8 保留(非数据删除)" \ + || fail "渲染归零形态断言失败" + +# --- 诚实口径:零 IR 不除零 · 零签署样本 pending · 零 needs-human p90=None --- +"$PY" - "$TMP/in.json" "$TMP/zero.json" <<'PYEOF' +import json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +d.update({"ir_count_month": 0, "sign_durations_seconds": [], "needs_human_dwell_hours": []}) +json.dump(d, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False) +PYEOF +OUT3=$("$PY" "$M" eval --input "$TMP/zero.json" 2>/dev/null) || { echo "::error::zero eval 失败"; exit 2; } +[[ $(jget "$OUT3" "d['metrics']['cost']['per_ir_usd']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['sign_p90_seconds']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['needs_human_p90_hours']") == None \ + && $(jget "$OUT3" "d['metrics']['attention']['suspicious_fast_signs']") == 0 ]] \ + && pass "零 IR/零样本:null 不除零不出假值(可疑签署 0=真 0 非缺数据)" \ + || fail "零分母诚实口径断言失败" + +# --- JSON schema:metrics 四组键全集(agent 消费契约) --- +"$PY" -c " +import json, sys +d = json.loads(sys.argv[1]) +ms = d['metrics'] +assert set(ms) == {'attention', 'security', 'cost', 'user_results'}, ms.keys() +assert {'sign_count','sign_p50_seconds','sign_p90_seconds','suspicious_fast_signs', + 'needs_human_count','needs_human_p90_hours','needs_human_p90_stop'} <= set(ms['attention']) +assert {'false_allow_window','false_deny_window','drill_red','drill_denom'} <= set(ms['security']) +assert {'per_ir_usd','actions_minutes_month','llm_tokens_month','butler_usd_week','patrol_yield'} <= set(ms['cost']) +assert {'products','quarterly_hard_quota'} == set(ms['user_results']) +assert d['generated_at'] == '2026-08-22T00:00:00Z' +" "$JSON" 2>/dev/null && pass "payload schema v2 键全集锁定" || fail "schema 断言失败" + +echo "== test-metrics-groups: pass=$PASS fail=$FAIL ==" +[[ $FAIL -eq 0 ]] From f4ec90d2ee048699daaa859c9fd7d9ade7f55bf5 Mon Sep 17 00:00:00 2001 From: randypanding Date: Sat, 22 Aug 2026 04:16:02 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat(dashboard):=20=E9=87=87=E9=9B=86?= =?UTF-8?q?=E5=B1=82=E7=BA=AF=E5=87=BD=E6=95=B0=E5=8C=BA+=E6=8F=90?= =?UTF-8?q?=E5=8F=96=E5=BC=8F=E8=87=AA=E6=B5=8B=EF=BC=88W5-C4=20.github#22?= =?UTF-8?q?7=EF=BC=8CADR-0073=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 逃逸双窗分割/演习红率口径(seed-drill 同 drill.py)/误放行台账窗过滤/ 签署 timeline 差/needs-human 停留/用户结果读取位——@w5c4-pure 标记对 提取自测 20 项(零网络)。PR 4/7。Card: Cloudbird-Software/.github#227 --- governance/dashboard-update.py | 159 ++++++++++++++++++++++++ governance/tests/test-metrics-wiring.sh | 148 ++++++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 governance/tests/test-metrics-wiring.sh diff --git a/governance/dashboard-update.py b/governance/dashboard-update.py index db80eae..3679e4f 100644 --- a/governance/dashboard-update.py +++ b/governance/dashboard-update.py @@ -196,6 +196,165 @@ def sli_stuck(repos): return stuck +# @w5c4-pure-begin —— 纯函数区(governance/tests/test-metrics-wiring.sh 按标记对 +# 提取本块离线单测——不复制实现,防"测试测影子";标记对缺失=测试红) +DRILL_MARKS = ("演练", "演习", "[drill]") # 演习数据约定标记(sli-report"演练"+ADR-0069"演习"双词兼容) + + +def _ts(s): + """ISO→datetime(失败 None);本块自包含(不依赖模块级 _iso——提取测试可独立运行)。""" + try: + return _dt.datetime.fromisoformat(str(s or "").replace("Z", "+00:00")) + except ValueError: + return None + + +def _is_drill_text(*texts): + joined = " ".join(str(t or "") for t in texts) + return any(m in joined for m in DRILL_MARKS) + + +def partition_escapes(prs, p0s, now): + """双窗逃逸事件:current=[now-7d,now),previous=[now-14d,now-7d)。 + + prs=merged PR 节点([auto-revert] 标题约定);p0s=post-merge 冒烟 P0 issue。 + 演习数据(title/body 含约定标记)从分子排除且 drills_excluded 计数可见—— + 过滤不可见=作弊通道(sli-report 先例)。reverts_current 供回滚率护栏(分子 + 只算 revert,不含 P0)。 + """ + w = _dt.timedelta(days=7) + cur = prev = drills = reverts_cur = 0 + for p in prs or []: + if "[auto-revert]" not in str(p.get("title") or ""): + continue + if _is_drill_text(p.get("title"), p.get("body")): + drills += 1 + continue + ts = _ts(p.get("mergedAt")) + if ts and now - 2 * w < ts <= now: + if ts > now - w: + cur += 1 + reverts_cur += 1 + else: + prev += 1 + for i in p0s or []: + if _is_drill_text(i.get("title"), i.get("body")): + drills += 1 + continue + ts = _ts(i.get("created_at")) + if ts and now - 2 * w < ts <= now: + if ts > now - w: + cur += 1 + else: + prev += 1 + return {"current": cur, "previous": prev, "reverts_current": reverts_cur, + "drills_excluded": drills} + + +def drill_redrate_lines(lines): + """drill history.jsonl 行→红率输入(kind=seed-drill,red/denom=red+green—— + 与 drill.py redrate 同口径,no-surface 与 failclose 演习不入分母;畸形行计入 bad 可见)。""" + red = denom = bad = 0 + for ln in lines or []: + ln = ln.strip() + if not ln or ln.startswith("#"): + continue + try: + rec = json.loads(ln) + except ValueError: + bad += 1 + continue + if rec.get("kind") != "seed-drill": + continue + v = rec.get("verdict") + if v in ("red", "green"): + denom += 1 + if v == "red": + red += 1 + return {"red": red, "denom": denom, "bad_lines": bad} + + +def false_decision_parse(text, now, window_days): + """arbiter 误放行台账文本→(窗内 false-allow 数, 窗内 false-deny 数, 全部行列表)。 + `#` 注释行跳过(台账文件头约定);date 出窗不计。""" + allow = deny = 0 + lines = [] + for ln in str(text or "").splitlines(): + ln = ln.strip() + if not ln or ln.startswith("#"): + continue + try: + rec = json.loads(ln) + except ValueError: + continue + lines.append(rec) + ts = _ts(rec.get("date")) + if ts and (now - ts).days <= window_days: + if rec.get("kind") == "false-allow": + allow += 1 + elif rec.get("kind") == "false-deny": + deny += 1 + return allow, deny, lines + + +def sign_durations(timelines): + """type:intent issue 的 timeline 事件列表→(签署耗时秒列表, 在途 draft 数)。 + + 耗时=首个 labeled state:ir-signed 时刻 − 首个 labeled state:ir-draft 时刻 + (宪法 §7:签署耗时如实计入判断预算)。无 draft 事件的已签 IR 不可算→跳过 + (不造 0);有 draft 无 signed→在途。""" + durations, in_flight = [], 0 + for events in timelines or []: + draft = signed = None + for ev in events or []: + if ev.get("event") != "labeled": + continue + name = (ev.get("label") or {}).get("name") + ts = _ts(ev.get("created_at")) + if not ts: + continue + if name == "state:ir-draft" and draft is None: + draft = ts + elif name == "state:ir-signed" and signed is None and draft is not None: + signed = ts + if draft and signed and signed > draft: + durations.append(round((signed - draft).total_seconds())) + elif draft and not signed: + in_flight += 1 + return durations, in_flight + + +def dwell_hours(events, now, label="state:needs-human"): + """timeline 事件→进入 label 态至今停留小时数(取最近一次 labeled 时刻—— + 反复进出取当前段)。无该事件→None。""" + latest = None + for ev in events or []: + if ev.get("event") != "labeled": + continue + if (ev.get("label") or {}).get("name") != label: + continue + ts = _ts(ev.get("created_at")) + if ts and (latest is None or ts > latest): + latest = ts + if latest is None or latest > now: + return None + return round((now - latest).total_seconds() / 3600, 2) + + +def user_metric_from(content_text): + """产品仓 user-result.yaml 文本→指标 dict(缺 metric_key/value=不完整→None)。 + 本地 import yaml——提取测试独立运行不依赖模块级导入。""" + import yaml + try: + d = yaml.safe_load(content_text) + except Exception: + return None + if isinstance(d, dict) and d.get("metric_key") and "value" in d: + return d + return None +# @w5c4-pure-end + + def build_payload(repos, cards, purl=""): rate, denom = sli_automerge(repos) sli = {"automerge_rate": rate, "human_touch_per_pr": None, "escape_rate": None, diff --git a/governance/tests/test-metrics-wiring.sh b/governance/tests/test-metrics-wiring.sh new file mode 100644 index 0000000..3a6f654 --- /dev/null +++ b/governance/tests/test-metrics-wiring.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# test-metrics-wiring.sh —— dashboard 采集层纯函数自测(W5-C4 .github#227,ADR-0073) +# +# 从 dashboard-update.py 按 @w5c4-pure 标记对提取纯函数区(不复制实现——防"测试 +# 测影子",test-ir0002.sh 同模式;标记对缺失=fail-closed 红),fixture 断言: +# 逃逸双窗分割(sustained 无状态化)· 演习过滤可见 · 演习红率分母口径 +# 误放行台账窗过滤 · 签署耗时 timeline 差(无 draft 不造 0)· needs-human 停留 +# 产品仓用户结果指标读取位(缺失/畸形=pending 不造数) +# 用法:bash governance/tests/test-metrics-wiring.sh(零网络零真实 gh) +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GOV="$(cd "$HERE/.." && pwd)" + +PASS=0; FAIL=0 +pass() { PASS=$((PASS+1)); echo "PASS $1"; } +fail() { FAIL=$((FAIL+1)); echo "FAIL $1"; } + +PY="" +for c in "${PYTHON:-}" python3 python py -3; do + [[ -n "$c" ]] || continue + "$c" -c 'import sys, yaml; print("ok")' >/dev/null 2>&1 || continue + PY="$c"; break +done +[[ -n "$PY" ]] || { echo "::error::无可用 python(含 pyyaml)"; exit 2; } + +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +SRC="$GOV/dashboard-update.py" +[[ -f "$SRC" ]] || { echo "FATAL: dashboard-update.py 不存在"; exit 2; } + +# --- 提取被测纯函数区(标记对缺失=fail-closed) --- +awk '/@w5c4-pure-begin/{f=1} f{print} /@w5c4-pure-end/{exit}' "$SRC" >"$TMP/pure_body.py" +if ! grep -q '^def partition_escapes(' "$TMP/pure_body.py"; then + echo "FATAL: 标记对内未找到纯函数定义(提取失效——实现与测试脱钩)"; exit 2 +fi +# 垫片头:提取块只依赖 stdlib(datetime/json)+局部 yaml——补导入即可独立运行 +{ echo 'import datetime as _dt'; echo 'import json'; cat "$TMP/pure_body.py"; } >"$TMP/pure.py" + +cat >>"$TMP/pure.py" <<'PYEOF' + +# ==== 驱动断言(now 固定——离线可复现,owner 复算同款) ==== +NOW = _dt.datetime(2026, 8, 22, 0, 0, tzinfo=_dt.timezone.utc) +W7 = _dt.timedelta(days=7) +results = [] + +def check(name, cond): + results.append((name, bool(cond))) + +# 1) 逃逸双窗分割:current=[now-7d,now) previous=[now-14d,now-7d);演习排除且可见 +prs = [ + {"title": "[auto-revert] #11 bad fix", "body": "x", + "mergedAt": (NOW - _dt.timedelta(days=1)).isoformat()}, # 本窗 revert + {"title": "[auto-revert] #10 older", "body": "x", + "mergedAt": (NOW - _dt.timedelta(days=10)).isoformat()}, # 上一窗 revert + {"title": "[auto-revert] drill tail", "body": "演习收尾", + "mergedAt": (NOW - _dt.timedelta(days=2)).isoformat()}, # 演习排除 + {"title": "feat: normal", "body": "x", + "mergedAt": (NOW - _dt.timedelta(days=2)).isoformat()}, # 非 revert + {"title": "[auto-revert] ancient", "body": "x", + "mergedAt": (NOW - _dt.timedelta(days=30)).isoformat()}, # 出 14d 窗 +] +p0s = [ + {"title": "post-merge 冒烟失败 #91", "created_at": (NOW - _dt.timedelta(days=3)).isoformat()}, + {"title": "post-merge 冒烟失败 #80", "created_at": (NOW - _dt.timedelta(days=9)).isoformat()}, + {"title": "post-merge 冒烟失败 drill", "body": "[drill]", "created_at": (NOW - _dt.timedelta(days=1)).isoformat()}, +] +esc = partition_escapes(prs, p0s, NOW) +check("逃逸双窗 current=2(1 revert+1 P0)", esc["current"] == 2) +check("逃逸双窗 previous=2(1 revert+1 P0)", esc["previous"] == 2) +check("回滚率分子 reverts_current=1(不含 P0)", esc["reverts_current"] == 1) +check("演习排除计数可见=2(过滤不可见=作弊通道)", esc["drills_excluded"] == 2) +check("空输入零窗", partition_escapes([], [], NOW) == {"current": 0, "previous": 0, "reverts_current": 0, "drills_excluded": 0}) + +# 2) 演习红率:kind=seed-drill + no-surface 不入分母(drill.py redrate 同口径);畸形行计数可见 +lines = [ + '# 台账头注释', + '{"kind":"seed-drill","verdict":"red"}', + '{"kind":"seed-drill","verdict":"green"}', + '{"kind":"seed-drill","verdict":"no-surface"}', + '{"kind":"failclose-drill","outcome":"pass"}', # 非种子演习——不入红率分母 + 'not-json-line', +] +agg = drill_redrate_lines(lines) +check("演习红率 red=1 denom=2(no-surface 与 failclose 出分母)", agg["red"] == 1 and agg["denom"] == 2) +check("畸形行 bad_lines=1 可见", agg["bad_lines"] == 1) + +# 3) 误放行台账:注释跳过 · 窗过滤 · infra 不算误拒(ADR-0054 §7) +ledger = '\n'.join([ + '# arbiter 误放行/误拒台账(false decision ledger,ADR-0054 §7)', + '{"date": "2026-08-15T00:00:00Z", "kind": "false-allow"}', + '{"date": "2026-08-20T00:00:00Z", "kind": "false-deny"}', + '{"date": "2026-05-01T00:00:00Z", "kind": "false-allow"}', + '{"date": "2026-08-21T00:00:00Z", "kind": "infra"}', +]) +allow, deny, fd_lines = false_decision_parse(ledger, NOW, 30) +check("误放行窗内=1(窗外不计)", allow == 1) +check("误拒窗内=1(infra 不计)", deny == 1) +check("行列表=4(注释行剔除)", len(fd_lines) == 4) + +# 4) 签署耗时:draft→signed timeline 差;在途计数;无 draft 不造 0 +t_ok = [ + {"event": "labeled", "label": {"name": "state:ir-draft"}, "created_at": "2026-08-20T10:00:00Z"}, + {"event": "labeled", "label": {"name": "state:ir-signed"}, "created_at": "2026-08-20T11:30:00Z"}, +] +t_fast = [ + {"event": "labeled", "label": {"name": "state:ir-draft"}, "created_at": "2026-08-21T10:00:00Z"}, + {"event": "labeled", "label": {"name": "state:ir-signed"}, "created_at": "2026-08-21T10:00:30Z"}, +] +t_inflight = [{"event": "labeled", "label": {"name": "state:ir-draft"}, "created_at": "2026-08-18T00:00:00Z"}] +t_nodraft = [{"event": "labeled", "label": {"name": "state:ir-signed"}, "created_at": "2026-08-19T00:00:00Z"}] +durs, inflight = sign_durations([t_ok, t_fast, t_inflight, t_nodraft]) +check("签署耗时=[5400, 30](90min 与 30s 快签)", durs == [5400, 30]) +check("在途 draft=1", inflight == 1) +check("signed 无 draft 不入统计(不造 0)", len(durs) == 2) + +# 5) needs-human 停留:取最近一次 labeled 时刻(反复进出取当前段) +ev = [ + {"event": "labeled", "label": {"name": "state:needs-human"}, "created_at": "2026-08-10T00:00:00Z"}, + {"event": "labeled", "label": {"name": "state:in-progress"}, "created_at": "2026-08-15T00:00:00Z"}, + {"event": "labeled", "label": {"name": "state:needs-human"}, "created_at": "2026-08-21T00:00:00Z"}, +] +h = dwell_hours(ev, NOW) +check("停留=24h(最近一次 needs-human)", h == 24.0) +check("无事件→None", dwell_hours([], NOW) is None) +check("未来时戳→None(不造负数)", dwell_hours( + [{"event": "labeled", "label": {"name": "state:needs-human"}, "created_at": "2026-08-23T00:00:00Z"}], NOW) is None) + +# 6) 用户结果读取位:完整→ok;缺 value→None;非 yaml→None(pending 不造数) +check("完整声明→dict", user_metric_from("metric_key: dau\nvalue: 42\nunit: 人\n") == {"metric_key": "dau", "value": 42, "unit": "人"}) +check("缺 metric_key→None", user_metric_from("value: 42\n") is None) +check("畸形 yaml→None", user_metric_from(":::not yaml[") is None) +check("空文本→None", user_metric_from("") is None) + +bad = [n for n, ok in results if not ok] +for n, ok in results: + print(("PASS " if ok else "FAIL ") + n) +raise SystemExit(1 if bad else 0) +PYEOF + +if "$PY" "$TMP/pure.py" >"$TMP/run.txt" 2>"$TMP/err.txt"; then + sed 's/^/ /' "$TMP/run.txt" + pass "dashboard 采集层纯函数 fixture 全过($(grep -c '^PASS' "$TMP/run.txt") 项)" +else + sed 's/^/ /' "$TMP/run.txt" "$TMP/err.txt" 2>/dev/null + fail "采集层纯函数断言失败(详见上行)" +fi + +echo "== test-metrics-wiring: pass=$PASS fail=$FAIL ==" +[[ $FAIL -eq 0 ]]