From 8c5882823719acad70cf48b61ee9c3eb799828be Mon Sep 17 00:00:00 2001 From: William Date: Thu, 23 Apr 2026 17:36:52 +0800 Subject: [PATCH 1/3] Implement Evolution Engine V1 --- .gitignore | 89 +++ README.md | 462 +++++++++++- README.zh.md | 452 +++++++++++ examples/.env.example | 66 ++ examples/dashboard/metrics.json | 91 +++ examples/dashboard/openclaw_config_hint.json | 30 + examples/evolution.yml | 83 ++ examples/prompts/actor_system.md | 16 + examples/prompts/judge_system.md | 29 + examples/scripts/status.sh | 3 + prompts/actor_system.md | 16 + prompts/judge_system.md | 29 + pyproject.toml | 45 ++ src/__init__.py | 2 + src/actor.py | 92 +++ src/cli.py | 240 ++++++ src/config_loader.py | 211 ++++++ src/github_client.py | 182 +++++ src/hard_stops.py | 111 +++ src/history.py | 30 + src/judge.py | 124 +++ src/llm.py | 174 +++++ src/observer.py | 109 +++ src/router.py | 38 + tests/__init__.py | 0 tests/setup_it001.sh | 202 +++++ tests/test_v1.py | 749 +++++++++++++++++++ 27 files changed, 3656 insertions(+), 19 deletions(-) create mode 100644 .gitignore create mode 100644 README.zh.md create mode 100644 examples/.env.example create mode 100644 examples/dashboard/metrics.json create mode 100644 examples/dashboard/openclaw_config_hint.json create mode 100644 examples/evolution.yml create mode 100644 examples/prompts/actor_system.md create mode 100644 examples/prompts/judge_system.md create mode 100755 examples/scripts/status.sh create mode 100644 prompts/actor_system.md create mode 100644 prompts/judge_system.md create mode 100644 pyproject.toml create mode 100644 src/__init__.py create mode 100644 src/actor.py create mode 100644 src/cli.py create mode 100644 src/config_loader.py create mode 100644 src/github_client.py create mode 100644 src/hard_stops.py create mode 100644 src/history.py create mode 100644 src/judge.py create mode 100644 src/llm.py create mode 100644 src/observer.py create mode 100644 src/router.py create mode 100644 tests/__init__.py create mode 100755 tests/setup_it001.sh create mode 100644 tests/test_v1.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93f9bed --- /dev/null +++ b/.gitignore @@ -0,0 +1,89 @@ +# =========================================================================== +# Evolution Engine — .gitignore +# =========================================================================== + +# --------------------------------------------------------------------------- +# 敏感文件 / Secrets +# --------------------------------------------------------------------------- +examples/.env +# .env.example 保留（模板，不含真实密钥） + +# --------------------------------------------------------------------------- +# 运行时状态 / Runtime state （每次运行由引擎自动生成，不应纳入版本控制） +# --------------------------------------------------------------------------- +**/.evolution_state.json +**/evolution_history.jsonl +# 如需保留示例历史日志，可注释上面一行 + +# --------------------------------------------------------------------------- +# Python +# --------------------------------------------------------------------------- +__pycache__/ +*.py[cod] +*$py.class +*.so + +# 构建产物 +dist/ +build/ +*.egg-info/ +*.egg +MANIFEST + +# 虚拟环境 +.venv/ +venv/ +env/ +ENV/ + +# pip +pip-log.txt +pip-delete-this-directory.txt + +# --------------------------------------------------------------------------- +# 测试 / Testing +# --------------------------------------------------------------------------- +.pytest_cache/ +.coverage +.coverage.* +htmlcov/ +.tox/ +nosetests.xml +coverage.xml +*.cover + +# --------------------------------------------------------------------------- +# 类型检查 / Type checking +# --------------------------------------------------------------------------- +.mypy_cache/ +.dmypy.json +.pyright/ + +# --------------------------------------------------------------------------- +# IDE / Editor +# --------------------------------------------------------------------------- +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# --------------------------------------------------------------------------- +# macOS +# --------------------------------------------------------------------------- +.DS_Store +.AppleDouble +.LSOverride + +# --------------------------------------------------------------------------- +# Windows +# --------------------------------------------------------------------------- +Thumbs.db +ehthumbs.db +Desktop.ini + +# --------------------------------------------------------------------------- +# 日志 / Logs +# --------------------------------------------------------------------------- +*.log +logs/ diff --git a/README.md b/README.md index dec742d..128ce5a 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,449 @@ -# Evolution Engine V1 Docs -自我进化引擎插件，万物皆可进化，与 ClawOSS V1 改造任务的需求文档、契约文档与测试用例。 +# Evolution Engine V1 -本仓库用于发布以下任务文档： +**Self-purification engine for autonomous codebases.** -- `PRD_EvolutionEngine.md` - - 通用净化引擎的产品需求文档 -- `PRD_ClawOSS_Evolvable.md` - - ClawOSS 改造为可进化系统的产品需求文档 -- `CONTRACT_metrics_schema.md` - - 净化引擎与 ClawOSS 的共享契约 -- `TEST_CASES.md` - - V1 验收测试用例 +Read an `evolution.yml`, observe evidence, let an Actor LLM propose a minimal patch, have an independent Judge LLM audit it, and automatically open a GitHub PR — all without local `git` or `gh` CLI. -## V1 目标 +> 🇨🇳 **[中文文档 → README.zh.md](README.zh.md)** -在 1 天内完成一个可用版本，使 ClawOSS 具备初步进化能力： -- ClawOSS 能稳定运行并输出生命体征 -- 净化引擎能读取配置和证据 -- Actor 与 Judge 独立工作 -- 在发现问题后自动提出可合并的 PR +--- -## 交付标准 +## Table of Contents -以 `TEST_CASES.md` 中的 `UNION-V1-001` 作为最终验收标准。 +- [How It Works](#how-it-works) +- [Quick Start](#quick-start) +- [Configuration Reference](#configuration-reference) + - [mission](#mission) + - [principles](#principles) + - [resources](#resources) + - [evidence\_sources](#evidence_sources) + - [hard\_stops](#hard_stops) + - [models](#models) + - [github](#github) + - [safety\_mode](#safety_mode) +- [Environment Variables](#environment-variables) + - [LLM Providers](#llm-providers) + - [GitHub Token](#github-token) +- [CLI Commands](#cli-commands) +- [Project Structure](#project-structure) +- [Running Tests](#running-tests) +- [Troubleshooting](#troubleshooting) +- [Documentation](#documentation) + +--- + +## How It Works + +``` +┌─────────────┐ evidence ┌───────────┐ patch ┌──────────┐ verdict ┌──────────┐ +│ Observer │ ────────────► │ Actor │ ─────────► │ Judge │ ──────────► │ Router │ +│ (read files,│ │ (LLM #1, │ │ (LLM #2, │ │ │ +│ run scripts│ │ proposes │ │ audits, │ PASS ──► create PR │ +│ fetch URLs)│ │ unified │ │ scores) │ │ via HTTP │ +└─────────────┘ │ diff) │ └──────────┘ FAIL ──► skip + log│ + └───────────┘ │ └──────────┘ + ▼ + Hard Stops + (budget / failures / + daily iterations) +``` + +Each run performs **one full cycle**: + +1. **Observer** — reads all `evidence_sources` (local files, shell scripts, remote URLs) and assembles a structured evidence bundle. +2. **Actor** — calls LLM #1 with the evidence + mission + principles. The model must return a single unified diff (fenced in ` ```diff `). +3. **Judge** — calls LLM #2 in a **completely fresh session** (no shared context with Actor). Receives only: mission, principles, evidence, and the patch text. Returns a structured JSON verdict: `PASS/FAIL`, `overall_score` (0–100), `confidence`, `top_risks`. +4. **Router** — on `PASS`: uses the GitHub REST API to create a branch, commit the patch, and open a PR. On `FAIL`: records the event and exits cleanly. +5. **Hard Stops** — a persistent circuit breaker that halts the engine if budget, consecutive failures, or daily iteration limits are exceeded. + +> **Actor / Judge isolation is a hard requirement.** The two LLMs must use different models or different API sessions. The Judge never sees the Actor's reasoning — only the final patch. This is enforced at runtime. + +--- + +## Quick Start + +### 1. Install + +```bash +# Clone and enter the repo +git clone && cd evolution-engine + +# Create a virtual environment +python3 -m venv .venv && source .venv/bin/activate + +# Install with dev dependencies +pip install -e ".[dev]" +``` + +### 2. Configure environment variables + +```bash +# Copy the template +cp examples/.env.example examples/.env + +# Edit examples/.env and fill in real values: +# ACTOR_MODEL, ACTOR_API_BASE_URL, ACTOR_API_KEY +# JUDGE_MODEL, JUDGE_API_BASE_URL, JUDGE_API_KEY +# GITHUB_REPO (e.g. "your-org/your-repo") +# GITHUB_TOKEN (GitHub Personal Access Token) +``` + +Load the variables (the `set -a` ensures they are exported to child processes): + +```bash +set -a && source examples/.env && set +a +``` + +### 3. Validate your config + +```bash +evolve validate --config examples/evolution.yml +``` + +A clean exit (code 0) means the YAML schema is valid. + +### 4. Run one evolution cycle + +```bash +evolve run --config examples/evolution.yml +``` + +If the Judge passes the patch, a real GitHub PR is created and the URL is printed to stdout. + +--- + +## Configuration Reference + +All configuration lives in a single `evolution.yml` file. See [`examples/evolution.yml`](examples/evolution.yml) for a complete working example. + +### `mission` + +A 3–5 sentence north-star statement that describes **what the codebase should achieve**. Both Actor and Judge receive this as the primary objective. + +```yaml +mission: | + Run a fleet of GitHub accounts that continuously contribute useful PRs + to high-quality open-source projects. Accounts must remain alive and + accumulate genuine reputation without triggering platform risk controls. +``` + +--- + +### `principles` + +An ordered list of rules the Judge uses to score every patch. Lower `priority` number = higher importance. The Judge must never pass a patch that violates a higher-priority rule to satisfy a lower-priority one. + +```yaml +principles: + - priority: 1 + rule: "Accounts must never be banned or shadowbanned" + - priority: 2 + rule: "Behavior must appear human; no mechanical patterns" + - priority: 3 + rule: "Every PR must provide genuine value to the target project" +``` + +--- + +### `resources` + +Declares paths to operational resources. Paths can be local file paths or `env:VAR_NAME` references. + +```yaml +resources: + accounts: + path: "./workspace/resources/accounts.enc.json" + health_check: "./scripts/check_account_health.sh" + proxies: + path: "env:PROXY_PROVIDER_URL" # read from environment variable + health_check: "./scripts/check_proxy_latency.sh" + budget: + daily_usd: 30 + hard_cap_usd: 100 +``` + +> **V1 note:** `resources` is declared for schema completeness. Active health-check invocation is a V2 feature. + +--- + +### `evidence_sources` + +A list of sources the Observer reads each cycle to build the evidence bundle. Three source types are supported: + +| Type | Example | Description | +|------|---------|-------------| +| Local file | `"./dashboard/metrics.json"` | Read and include file content | +| Shell script | `"./scripts/status.sh"` | Execute and capture stdout | +| Remote URL | `"https://example.com/api/status"` | HTTP GET and include response body | + +```yaml +evidence_sources: + - "./dashboard/metrics.json" + - "./dashboard/config_hint.json" # current config excerpt to help Actor produce exact diffs + - "./scripts/status.sh" +``` + +**Tip:** Include a `config_hint.json` that contains the exact current content of the file you want the Actor to patch. This dramatically improves diff precision and Judge pass rates. + +--- + +### `hard_stops` + +A persistent circuit breaker. State is saved to `.evolution_state.json` next to the config file. + +```yaml +hard_stops: + budget_hard_cap_usd: 100 # halt if cumulative LLM spend exceeds this + max_consecutive_failures: 5 # halt after N consecutive Judge FAILs or errors + max_iterations_per_day: 20 # halt after N cycles in a single calendar day + on_trigger: "halt_and_notify" # write a message to stderr and exit non-zero +``` + +To clear a halted state after human review: + +```bash +evolve reset --config examples/evolution.yml +``` + +`reset` clears the halted flag **and** resets the consecutive-failure counter so a single stale failure doesn't immediately re-trigger. + +--- + +### `models` + +Configures the Actor and Judge LLMs. Two formats are supported: + +**String shorthand** (Anthropic native SDK, key from `ANTHROPIC_API_KEY`): + +```yaml +models: + actor: "claude-sonnet-4" + judge: "claude-opus-4" +``` + +**Full object form** (any OpenAI-compatible API): + +```yaml +models: + actor: + name: "env:ACTOR_MODEL" # resolved from $ACTOR_MODEL at runtime + api_base_url: "env:ACTOR_API_BASE_URL" + api_key_env: "ACTOR_API_KEY" # name of the env var holding the key + judge: + name: "env:JUDGE_MODEL" + api_base_url: "env:JUDGE_API_BASE_URL" + api_key_env: "JUDGE_API_KEY" +``` + +`name` and `api_base_url` support the `env:VAR_NAME` prefix — the real value is resolved at call time, keeping the YAML file secret-free. + +**Common `api_base_url` values:** + +| Provider | `api_base_url` | +|----------|---------------| +| OpenAI | `https://api.openai.com/v1` | +| DeepSeek | `https://api.deepseek.com/v1` | +| Groq | `https://api.groq.com/openai/v1` | +| OpenRouter | `https://openrouter.ai/api/v1` | +| Ollama (local) | `http://localhost:11434/v1` | +| Anthropic native | *(omit `api_base_url`)* | + +> Actor and Judge can use **different providers**. Recommend a strong coding model for Actor and a strong reasoning model for Judge. + +--- + +### `github` + +Target repository for PR creation. The Router calls the GitHub REST API directly — **no local `git` or `gh` CLI required**. + +```yaml +github: + repo: "env:GITHUB_REPO" # "owner/repo" or env:VAR_NAME + token_env: "GITHUB_TOKEN" # env var holding the Personal Access Token + base_branch: "main" # branch to create PRs against +``` + +Required GitHub PAT permissions: **`repo`** (full) or at minimum **`contents: write`** + **`pull_requests: write`**. + +Get a token at: https://github.com/settings/tokens + +--- + +### `safety_mode` + +V1 only supports `"human_in_the_loop"`. The engine always creates a PR for human review rather than auto-merging. + +```yaml +safety_mode: "human_in_the_loop" +``` + +--- + +## Environment Variables + +Copy [`examples/.env.example`](examples/.env.example) to `examples/.env` and fill in real values. + +```bash +cp examples/.env.example examples/.env +# edit examples/.env +set -a && source examples/.env && set +a +``` + +### LLM Providers + +| Variable | Description | +|----------|-------------| +| `ACTOR_MODEL` | Model name for the Actor (e.g. `gpt-4o`, `deepseek-chat`) | +| `ACTOR_API_BASE_URL` | API base URL for Actor's provider | +| `ACTOR_API_KEY` | API key for Actor | +| `JUDGE_MODEL` | Model name for the Judge | +| `JUDGE_API_BASE_URL` | API base URL for Judge's provider | +| `JUDGE_API_KEY` | API key for Judge | +| `ANTHROPIC_API_KEY` | Required only when using Anthropic native SDK (string shorthand form) | + +### GitHub Token + +| Variable | Description | +|----------|-------------| +| `GITHUB_REPO` | Target repo in `owner/repo` format | +| `GITHUB_TOKEN` | Personal Access Token with `repo` + `pull_requests` permissions | + +> **Security:** Never commit `examples/.env`. It is excluded by `.gitignore`. Only `examples/.env.example` (which contains no real secrets) should be committed. + +--- + +## CLI Commands + +```bash +# Validate evolution.yml schema (exits 0 if valid) +evolve validate --config examples/evolution.yml + +# Run one Observer → Actor → Judge → Router cycle +evolve run --config examples/evolution.yml + +# Run with verbose logging +evolve -v run --config examples/evolution.yml + +# Print the JSONL evolution history +evolve history --config examples/evolution.yml + +# Clear halted state after human review +evolve reset --config examples/evolution.yml +``` + +--- + +## Project Structure + +``` +evolution-engine/ +├── src/ +│ ├── cli.py # Click CLI entry point (validate / run / history / reset) +│ ├── config_loader.py # Pydantic schema validation for evolution.yml +│ ├── observer.py # Evidence source scanner (files / scripts / URLs) +│ ├── actor.py # Actor LLM caller + unified diff extractor +│ ├── judge.py # Independent Judge LLM + JSON verdict parser +│ ├── router.py # Post-verdict dispatcher (PR creation or skip) +│ ├── github_client.py # GitHub REST API: blob → tree → commit → branch → PR +│ ├── hard_stops.py # Persistent circuit breaker state machine +│ ├── llm.py # Unified LLM caller (Anthropic native + OpenAI-compat) +│ └── history.py # JSONL history writer +├── prompts/ +│ ├── actor_system.md # Actor system prompt +│ └── judge_system.md # Judge system prompt +├── tests/ +│ └── test_v1.py # 45 unit + integration tests (Task A acceptance suite) +├── examples/ +│ ├── evolution.yml # Sample config (ClawOSS use-case) +│ ├── .env.example # Environment variable template (commit this) +│ ├── .env # Real secrets — DO NOT COMMIT (git-ignored) +│ ├── scripts/ +│ │ └── status.sh # Example evidence script +│ └── dashboard/ +│ ├── metrics.json # Example metrics evidence file +│ └── openclaw_config_hint.json # Example config-hint evidence file +├── pyproject.toml +├── .gitignore +├── README.md # This file (English) +└── README.zh.md # Chinese version +``` + +--- + +## Running Tests + +```bash +source .venv/bin/activate +python -m pytest tests/ -v +``` + +All 45 Task A acceptance tests should pass. No real API keys or GitHub tokens are required — all LLM and GitHub calls are mocked. + +To run a specific test group: + +```bash +# Unit tests only +python -m pytest tests/ -v -k "TestHardStops or TestConfig or TestRouter" + +# Integration test only +python -m pytest tests/ -v -k "test_full_cycle" +``` + +--- + +## Troubleshooting + +**`Error: metrics.json stale: generated_at age Xs > 60s`** + +The Observer enforces a 60-second freshness window on metrics files. Regenerate the file or update its `generated_at` timestamp before running. + +**`ACTOR_API_KEY is not set`** + +Environment variables set with plain `source .env` are not exported to child processes. Always use: +```bash +set -a && source examples/.env && set +a +``` + +**`Hard stop triggered: max consecutive failures reached`** + +Previous failed runs accumulated in `.evolution_state.json`. Run: +```bash +evolve reset --config examples/evolution.yml +``` + +**`Cannot get branch 'main': Branch not found`** + +The target repository's default branch is not `main`. Check on GitHub and update `base_branch` in `evolution.yml`: +```yaml +github: + base_branch: "master" # or whatever the default branch is +``` + +**`Cannot access repo 'owner/repo'`** + +- Verify `GITHUB_REPO` is set to `owner/repo` format (not a full URL). +- Verify `GITHUB_TOKEN` has `repo` scope. +- If the repo is private, the token must belong to an account with access. + +**Judge verdict is always `FAIL`** + +- Ensure the Actor is producing a fenced diff block (` ```diff ... ``` `). +- Add a `config_hint.json` evidence source with the exact current file content so the Actor can produce a precise, context-matching diff. +- Increase `max_tokens` for the Judge if responses are being truncated. + +--- + +## Documentation + +| File | Description | +|------|-------------| +| [`PRD_EvolutionEngine.md`](PRD_EvolutionEngine.md) | Product requirements — Task A (this engine) | +| [`PRD_ClawOSS_Evolvable.md`](PRD_ClawOSS_Evolvable.md) | ClawOSS evolvable transformation — Task B | +| [`CONTRACT_metrics_schema.md`](CONTRACT_metrics_schema.md) | Shared data contract: `metrics.json` schema | +| [`TEST_CASES.md`](TEST_CASES.md) | V1 acceptance test cases | + +--- + +## License + +Apache 2.0 diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..cdb7c41 --- /dev/null +++ b/README.zh.md @@ -0,0 +1,452 @@ +# Evolution Engine V1 + +**面向自主代码库的自净化引擎。** + +读取一份 `evolution.yml`，观察证据，让 Actor LLM 提出最小化 patch，再由完全独立的 Judge LLM 审核，最终通过 GitHub REST API 自动提 PR——全程无需本地 `git` 或 `gh` 命令行工具。 + +> 🇺🇸 **[English Documentation → README.md](README.md)** + +--- + +## 目录 + +- [工作原理](#工作原理) +- [快速上手](#快速上手) +- [配置参考](#配置参考) + - [mission（任务目标）](#mission任务目标) + - [principles（审核原则）](#principles审核原则) + - [resources（资源声明）](#resources资源声明) + - [evidence\_sources（证据来源）](#evidence_sources证据来源) + - [hard\_stops（熔断器）](#hard_stops熔断器) + - [models（LLM 配置）](#modelsllm-配置) + - [github（PR 目标仓库）](#githubpr-目标仓库) + - [safety\_mode（安全模式）](#safety_mode安全模式) +- [环境变量](#环境变量) + - [LLM 服务商配置](#llm-服务商配置) + - [GitHub Token](#github-token) +- [CLI 命令](#cli-命令) +- [项目结构](#项目结构) +- [运行测试](#运行测试) +- [常见问题](#常见问题) +- [相关文档](#相关文档) + +--- + +## 工作原理 + +``` +┌─────────────┐ 证据 ┌───────────┐ patch ┌──────────┐ 裁定 ┌──────────┐ +│ Observer │ ──────────► │ Actor │ ─────────► │ Judge │ ───────────► │ Router │ +│ (读文件、 │ │ (LLM #1, │ │ (LLM #2, │ │ │ +│ 执行脚本、 │ │ 提出统一 │ │ 独立审核│ PASS ───► 调 GitHub │ +│ 请求 URL) │ │ diff) │ │ 打分) │ │ API 提 PR│ +└─────────────┘ └───────────┘ └──────────┘ FAIL ───► 跳过+记录│ + │ └──────────┘ + ▼ + Hard Stops + (预算 / 连续失败 / + 每日迭代上限) +``` + +每次运行执行**一个完整周期**： + +1. **Observer** — 读取所有 `evidence_sources`（本地文件、Shell 脚本、远程 URL），组装结构化证据包。 +2. **Actor** — 将证据 + mission + principles 发给 LLM #1，模型必须返回一个用 ` ```diff ` 包裹的统一 diff。 +3. **Judge** — 在**全新独立会话**中调用 LLM #2（与 Actor 完全隔离，不共享任何上下文）。Judge 只接收：mission、principles、evidence 和 patch 文本，返回结构化 JSON 裁定：`PASS/FAIL`、`overall_score`（0–100）、`confidence`、`top_risks`。 +4. **Router** — 裁定为 `PASS` 时：通过 GitHub REST API 创建分支、提交 patch、开 PR。裁定为 `FAIL` 时：记录事件，干净退出。 +5. **Hard Stops** — 持久化熔断器，预算超限、连续失败或每日迭代超限时自动暂停引擎。 + +> **Actor / Judge 隔离是硬性要求。** 两个 LLM 必须使用不同的模型或不同的 API 会话。Judge 永远只看最终 patch，看不到 Actor 的推理过程。这一点在运行时强制校验。 + +--- + +## 快速上手 + +### 1. 安装 + +```bash +# 克隆并进入目录 +git clone && cd evolution-engine + +# 创建虚拟环境 +python3 -m venv .venv && source .venv/bin/activate + +# 安装（含开发依赖） +pip install -e ".[dev]" +``` + +### 2. 配置环境变量 + +```bash +# 复制模板 +cp examples/.env.example examples/.env + +# 编辑 examples/.env，填入真实值： +# ACTOR_MODEL、ACTOR_API_BASE_URL、ACTOR_API_KEY +# JUDGE_MODEL、JUDGE_API_BASE_URL、JUDGE_API_KEY +# GITHUB_REPO（格式：owner/repo） +# GITHUB_TOKEN（GitHub 个人访问令牌） +``` + +加载环境变量（`set -a` 确保变量 export 给子进程，Python 才能读到）： + +```bash +set -a && source examples/.env && set +a +``` + +### 3. 校验配置文件 + +```bash +evolve validate --config examples/evolution.yml +``` + +退出码为 0 表示 YAML schema 合法。 + +### 4. 运行一次进化周期 + +```bash +evolve run --config examples/evolution.yml +``` + +如果 Judge 通过了 patch，会在 GitHub 上真实创建 PR，并将 PR 链接打印到 stdout。 + +--- + +## 配置参考 + +所有配置集中在一个 `evolution.yml` 文件中。完整示例见 [`examples/evolution.yml`](examples/evolution.yml)。 + +### `mission`（任务目标） + +3–5 句话描述**代码库应该实现什么目标**。Actor 和 Judge 都以此作为最高指令。 + +```yaml +mission: | + 运营一批 GitHub 账号，持续向真实的优质开源项目贡献有用的 PR。 + 账号长期存活并积累真实声誉，避免被平台风控识别。 +``` + +--- + +### `principles`（审核原则） + +Judge 评分每个 patch 时遵循的规则列表，按优先级排序。`priority` 数字越小，优先级越高。Judge 不能为了满足低优先级规则而违反高优先级规则。 + +```yaml +principles: + - priority: 1 + rule: "账号绝不能被封禁或 Shadowban" + - priority: 2 + rule: "行为必须拟人化，杜绝机械化特征" + - priority: 3 + rule: "PR 必须对目标项目有实质价值，禁止水 PR" + - priority: 4 + rule: "PR 被维护者真心合并或获得正面 review" + - priority: 5 + rule: "Token 与基础设施成本与产出价值匹配" +``` + +--- + +### `resources`（资源声明） + +声明操作资源的路径。路径可以是本地文件路径，也可以用 `env:变量名` 引用环境变量。 + +```yaml +resources: + accounts: + path: "./workspace/resources/accounts.enc.json" + health_check: "./scripts/check_account_health.sh" + proxies: + path: "env:PROXY_PROVIDER_URL" # 运行时从环境变量读取 + health_check: "./scripts/check_proxy_latency.sh" + budget: + daily_usd: 30 + hard_cap_usd: 100 +``` + +> **V1 说明：** `resources` 字段用于 schema 完整性声明。健康检查的主动调用是 V2 功能。 + +--- + +### `evidence_sources`（证据来源） + +Observer 每个周期读取的数据来源列表，用于构建证据包。支持三种类型： + +| 类型 | 示例 | 说明 | +|------|------|------| +| 本地文件 | `"./dashboard/metrics.json"` | 读取并包含文件内容 | +| Shell 脚本 | `"./scripts/status.sh"` | 执行并捕获 stdout | +| 远程 URL | `"https://example.com/api/status"` | HTTP GET 并包含响应体 | + +```yaml +evidence_sources: + - "./dashboard/metrics.json" + - "./dashboard/config_hint.json" # 目标文件的当前内容，帮助 Actor 生成精确 diff + - "./scripts/status.sh" +``` + +**技巧：** 加入一个 `config_hint.json`，包含你希望 Actor 修改的文件的当前真实内容。这能大幅提升 diff 精度，显著提高 Judge 通过率。 + +--- + +### `hard_stops`（熔断器） + +持久化熔断器，状态保存在配置文件同级的 `.evolution_state.json` 中。 + +```yaml +hard_stops: + budget_hard_cap_usd: 100 # 累计 LLM 花费超限则暂停 + max_consecutive_failures: 5 # 连续 N 次 Judge FAIL 或错误后暂停 + max_iterations_per_day: 20 # 单日运行超过 N 次后暂停 + on_trigger: "halt_and_notify" # 向 stderr 输出通知并以非零码退出 +``` + +人工排查后解除暂停： + +```bash +evolve reset --config examples/evolution.yml +``` + +`reset` 会同时清除暂停标志**和**连续失败计数器，避免残留计数立即再次触发熔断。 + +--- + +### `models`（LLM 配置） + +配置 Actor 和 Judge 使用的 LLM。支持两种写法： + +**字符串简写**（Anthropic 原生 SDK，key 来自 `ANTHROPIC_API_KEY`）： + +```yaml +models: + actor: "claude-sonnet-4" + judge: "claude-opus-4" +``` + +**完整对象形式**（任意 OpenAI 兼容 API）： + +```yaml +models: + actor: + name: "env:ACTOR_MODEL" # 运行时从 $ACTOR_MODEL 解析 + api_base_url: "env:ACTOR_API_BASE_URL" + api_key_env: "ACTOR_API_KEY" # 持有密钥的环境变量名 + judge: + name: "env:JUDGE_MODEL" + api_base_url: "env:JUDGE_API_BASE_URL" + api_key_env: "JUDGE_API_KEY" +``` + +`name` 和 `api_base_url` 支持 `env:变量名` 前缀——真实值在调用时才解析，YAML 文件中不含任何密钥。 + +**常用 `api_base_url`：** + +| 服务商 | `api_base_url` | +|--------|---------------| +| OpenAI | `https://api.openai.com/v1` | +| DeepSeek | `https://api.deepseek.com/v1` | +| Groq | `https://api.groq.com/openai/v1` | +| OpenRouter | `https://openrouter.ai/api/v1` | +| Ollama（本地） | `http://localhost:11434/v1` | +| Anthropic 原生 | *（不填 `api_base_url`）* | + +> Actor 和 Judge 可以**使用不同的服务商**。建议 Actor 用代码能力强的模型，Judge 用推理能力强的模型。 + +--- + +### `github`（PR 目标仓库） + +PR 创建的目标仓库。Router 直接调用 GitHub REST API——**无需本地 `git` 或 `gh` 命令行工具**。 + +```yaml +github: + repo: "env:GITHUB_REPO" # "owner/repo" 或 env:变量名 + token_env: "GITHUB_TOKEN" # 持有 PAT 的环境变量名 + base_branch: "main" # PR 的目标基础分支 +``` + +GitHub PAT 所需权限：**`repo`**（完整）或至少 **`contents: write`** + **`pull_requests: write`**。 + +获取 Token：https://github.com/settings/tokens + +--- + +### `safety_mode`（安全模式） + +V1 仅支持 `"human_in_the_loop"`。引擎始终创建 PR 供人工审核，不会自动合并。 + +```yaml +safety_mode: "human_in_the_loop" +``` + +--- + +## 环境变量 + +将 [`examples/.env.example`](examples/.env.example) 复制为 `examples/.env` 并填入真实值： + +```bash +cp examples/.env.example examples/.env +# 编辑 examples/.env +set -a && source examples/.env && set +a +``` + +### LLM 服务商配置 + +| 变量名 | 说明 | +|--------|------| +| `ACTOR_MODEL` | Actor 使用的模型名（如 `gpt-4o`、`deepseek-chat`） | +| `ACTOR_API_BASE_URL` | Actor 服务商的 API base URL | +| `ACTOR_API_KEY` | Actor 的 API 密钥 | +| `JUDGE_MODEL` | Judge 使用的模型名 | +| `JUDGE_API_BASE_URL` | Judge 服务商的 API base URL | +| `JUDGE_API_KEY` | Judge 的 API 密钥 | +| `ANTHROPIC_API_KEY` | 仅使用 Anthropic 原生 SDK（字符串简写形式）时需要 | + +### GitHub Token + +| 变量名 | 说明 | +|--------|------| +| `GITHUB_REPO` | 目标仓库，格式 `owner/repo` | +| `GITHUB_TOKEN` | 个人访问令牌，需含 `repo` + `pull_requests` 权限 | + +> **安全提示：** 永远不要提交 `examples/.env`。该文件已被 `.gitignore` 排除。只有 `examples/.env.example`（不含真实密钥）应该提交到版本控制。 + +--- + +## CLI 命令 + +```bash +# 校验 evolution.yml schema（退出码 0 表示合法） +evolve validate --config examples/evolution.yml + +# 运行一次 Observer → Actor → Judge → Router 周期 +evolve run --config examples/evolution.yml + +# 开启详细日志模式运行 +evolve -v run --config examples/evolution.yml + +# 打印 JSONL 格式的进化历史 +evolve history --config examples/evolution.yml + +# 人工排查后解除熔断暂停 +evolve reset --config examples/evolution.yml +``` + +--- + +## 项目结构 + +``` +evolution-engine/ +├── src/ +│ ├── cli.py # Click CLI 入口（validate / run / history / reset） +│ ├── config_loader.py # evolution.yml 的 Pydantic schema 校验 +│ ├── observer.py # 证据来源扫描器（文件 / 脚本 / URL） +│ ├── actor.py # Actor LLM 调用 + 统一 diff 提取 +│ ├── judge.py # 独立 Judge LLM + JSON 裁定解析 +│ ├── router.py # 裁定后分发器（创建 PR 或跳过） +│ ├── github_client.py # GitHub REST API：blob → tree → commit → branch → PR +│ ├── hard_stops.py # 持久化熔断器状态机 +│ ├── llm.py # 统一 LLM 调用层（Anthropic 原生 + OpenAI 兼容） +│ └── history.py # JSONL 历史写入器 +├── prompts/ +│ ├── actor_system.md # Actor 系统提示词 +│ └── judge_system.md # Judge 系统提示词 +├── tests/ +│ └── test_v1.py # 45 个单元 + 集成测试（Task A 验收套件） +├── examples/ +│ ├── evolution.yml # 示例配置（ClawOSS 场景） +│ ├── .env.example # 环境变量模板（应提交） +│ ├── .env # 真实密钥——禁止提交（已 git-ignore） +│ ├── scripts/ +│ │ └── status.sh # 示例证据脚本 +│ └── dashboard/ +│ ├── metrics.json # 示例指标证据文件 +│ └── openclaw_config_hint.json # 示例配置提示证据文件 +├── pyproject.toml +├── .gitignore +├── README.md # 英文文档 +└── README.zh.md # 本文件（中文） +``` + +--- + +## 运行测试 + +```bash +source .venv/bin/activate +python -m pytest tests/ -v +``` + +所有 45 个 Task A 验收测试应全部通过。无需真实 API 密钥或 GitHub Token——所有 LLM 和 GitHub 调用均已 mock。 + +运行特定测试组： + +```bash +# 仅运行单元测试 +python -m pytest tests/ -v -k "TestHardStops or TestConfig or TestRouter" + +# 仅运行集成测试 +python -m pytest tests/ -v -k "test_full_cycle" +``` + +--- + +## 常见问题 + +**`Error: metrics.json stale: generated_at age Xs > 60s`** + +Observer 对 metrics 文件有 60 秒的新鲜度校验。运行前更新文件的 `generated_at` 时间戳，或重新生成该文件。 + +**`ACTOR_API_KEY is not set`** + +用普通 `source .env` 设置的变量不会 export 给子进程，Python 读不到。必须使用： +```bash +set -a && source examples/.env && set +a +``` + +**`Hard stop triggered: max consecutive failures reached`** + +之前的失败运行已在 `.evolution_state.json` 中积累了计数。执行： +```bash +evolve reset --config examples/evolution.yml +``` + +**`Cannot get branch 'main': Branch not found`** + +目标仓库的默认分支不是 `main`。去 GitHub 确认分支名后更新 `evolution.yml`： +```yaml +github: + base_branch: "v6-release" # 填写实际的默认分支名 +``` + +**`Cannot access repo 'owner/repo'`** + +- 确认 `GITHUB_REPO` 格式为 `owner/repo`（不是完整 URL）。 +- 确认 `GITHUB_TOKEN` 拥有 `repo` scope。 +- 若仓库为私有，Token 所属账号需有访问权限。 + +**Judge 裁定总是 `FAIL`** + +- 确认 Actor 输出了完整的 diff 块（` ```diff ... ``` `）。 +- 添加 `config_hint.json` 证据来源，包含目标文件的当前真实内容，帮助 Actor 生成精确的、上下文匹配的 diff。 +- 如果响应被截断，适当增大 Judge 的 `max_tokens`。 + +--- + +## 相关文档 + +| 文件 | 说明 | +|------|------| +| [`PRD_EvolutionEngine.md`](PRD_EvolutionEngine.md) | 产品需求文档 — Task A（本引擎） | +| [`PRD_ClawOSS_Evolvable.md`](PRD_ClawOSS_Evolvable.md) | ClawOSS 可进化改造方案 — Task B | +| [`CONTRACT_metrics_schema.md`](CONTRACT_metrics_schema.md) | 共享数据契约：`metrics.json` schema | +| [`TEST_CASES.md`](TEST_CASES.md) | V1 验收测试用例 | + +--- + +## 许可证 + +Apache 2.0 diff --git a/examples/.env.example b/examples/.env.example new file mode 100644 index 0000000..838b7e5 --- /dev/null +++ b/examples/.env.example @@ -0,0 +1,66 @@ +# =========================================================================== +# Evolution Engine — 环境变量模板 +# 使用方法：cp .env.example .env 然后填入真实值 +# 运行：set -a && source .env && set +a && evolve run --config evolution.yml +# （必须用 set -a 确保变量被 export 给子进程，否则 Python 读不到） +# =========================================================================== + +# --------------------------------------------------------------------------- +# Actor LLM（代码提案角色） +# 负责分析 evidence 并生成修复 patch，建议使用代码能力强的模型 +# --------------------------------------------------------------------------- +ACTOR_MODEL=model-for-actor +ACTOR_API_BASE_URL=base-url-for-actor +ACTOR_API_KEY=sk-your-actor-api-key-here + +# --------------------------------------------------------------------------- +# Judge LLM（独立审核角色） +# 独立审核 Actor 的 patch，建议使用推理能力强的模型，且与 Actor 隔离 +# --------------------------------------------------------------------------- +JUDGE_MODEL=model-for-judge +JUDGE_API_BASE_URL=base-url-for-judge +JUDGE_API_KEY=sk-your-judge-api-key-here + +# --------------------------------------------------------------------------- +# 其他第三方服务商示例（取消注释并修改即可切换） +# --------------------------------------------------------------------------- + +# OpenAI +# ACTOR_MODEL=gpt-4o +# ACTOR_API_BASE_URL=https://api.openai.com/v1 +# ACTOR_API_KEY=sk-... + +# DeepSeek +# ACTOR_MODEL=deepseek-chat +# ACTOR_API_BASE_URL=https://api.deepseek.com/v1 +# ACTOR_API_KEY=sk-... + +# Groq (高速推理，适合 Judge) +# JUDGE_MODEL=llama-3.3-70b-versatile +# JUDGE_API_BASE_URL=https://api.groq.com/openai/v1 +# JUDGE_API_KEY=gsk_... + +# OpenRouter（同时接入多个服务商） +# ACTOR_MODEL=anthropic/claude-sonnet-4 +# JUDGE_MODEL=google/gemini-2.5-pro +# ACTOR_API_BASE_URL=https://openrouter.ai/api/v1 +# JUDGE_API_BASE_URL=https://openrouter.ai/api/v1 +# ACTOR_API_KEY=sk-or-... +# JUDGE_API_KEY=sk-or-... + +# Anthropic 原生（evolution.yml 中用字符串简写形式时需要此变量） +# ANTHROPIC_API_KEY=sk-ant-... + +# --------------------------------------------------------------------------- +# GitHub 配置（Router 直接调用 GitHub REST API，无需安装 git / gh CLI） +# GITHUB_REPO：目标仓库，格式 "owner/repo-name" +# GITHUB_TOKEN：Personal Access Token，权限需含 repo + pull_requests +# 获取 Token：https://github.com/settings/tokens +# --------------------------------------------------------------------------- +GITHUB_REPO=your-org/your-repo +GITHUB_TOKEN=ghp_your_github_token_here + +# --------------------------------------------------------------------------- +# 资源路径（如果 evolution.yml 中用了 env:PROXY_PROVIDER_URL） +# --------------------------------------------------------------------------- +# PROXY_PROVIDER_URL=https://your-proxy-provider.com/api/list diff --git a/examples/dashboard/metrics.json b/examples/dashboard/metrics.json new file mode 100644 index 0000000..c680188 --- /dev/null +++ b/examples/dashboard/metrics.json @@ -0,0 +1,91 @@ +{ + "schema_version": "1.0.0", + "generated_at": "2026-04-23T09:13:30Z", + "accounts": [ + { + "id": "clawoss_main", + "status": "degraded", + "status_detail": "subagent_cascade_failure", + "created_at": "2026-01-01T00:00:00Z", + "last_active_at": "2026-04-23T09:13:30Z", + "pr_count": 42, + "merge_count": 18 + } + ], + "prs": [], + "resources": { + "accounts_total": 1, + "accounts_alive": 1, + "accounts_rate_limited": 0, + "accounts_banned": 0, + "proxies_total": 1, + "proxies_healthy": 1 + }, + "budget": { + "daily_used_usd": 8.2, + "daily_cap_usd": 30.0, + "cumulative_used_usd": 28.0, + "hard_cap_usd": 100.0, + "reset_at": "2026-04-24T00:00:00Z" + }, + "recent_events": [ + { + "ts": "2026-04-23T09:13:30Z", + "type": "subagent_spawn_rate_limit", + "severity": "error", + "details": { + "spawned_count": 7, + "config_max_concurrent": 6, + "actual_safe_limit": 3, + "error": "HTTP 429 Too Many Requests \u2014 GitHub secondary rate limit triggered", + "affected_subagents": [ + "impl-1", + "impl-2", + "impl-3", + "impl-4", + "impl-5", + "impl-6", + "impl-7" + ], + "all_killed": true + } + }, + { + "ts": "2026-04-23T09:13:30Z", + "type": "subagent_spawn_rate_limit", + "severity": "error", + "details": { + "spawned_count": 6, + "config_max_concurrent": 6, + "actual_safe_limit": 3, + "error": "HTTP 429 Too Many Requests \u2014 GitHub secondary rate limit triggered", + "affected_subagents": [ + "impl-1", + "impl-2", + "impl-3", + "impl-4", + "impl-5", + "impl-6" + ], + "all_killed": true + } + }, + { + "ts": "2026-04-23T09:13:30Z", + "type": "config_mismatch_detected", + "severity": "warning", + "details": { + "file": "config/openclaw.json", + "json_path": "agents.defaults.subagents.maxConcurrent", + "current_value": 6, + "correct_value": 3, + "reason": "heartbeat prompt explicitly states maxConcurrent is 3; value 6 causes cascading 429 errors that kill ALL subagents in the cycle", + "heartbeat_quote": "maxConcurrent is 3. NEVER spawn more than 3 implementations per heartbeat cycle. Spawning 10+ causes API 429 rate limit errors that kill ALL sub-agents." + } + } + ], + "diagnostics": { + "root_cause": "config/openclaw.json sets agents.defaults.subagents.maxConcurrent=6 but the safe operational limit is 3. Every heartbeat cycle that attempts to use the full 6 slots triggers GitHub 429 rate limits, killing all concurrent subagents and wasting the entire cycle budget.", + "recommended_fix": "Change maxConcurrent from 6 to 3 in config/openclaw.json" + } +} \ No newline at end of file diff --git a/examples/dashboard/openclaw_config_hint.json b/examples/dashboard/openclaw_config_hint.json new file mode 100644 index 0000000..1a522ae --- /dev/null +++ b/examples/dashboard/openclaw_config_hint.json @@ -0,0 +1,30 @@ +{ + "_note": "Current content of config/openclaw.json in the target repo (AndrosEt/ClawOSS). Used by Actor to produce an exact diff.", + "target_file": "config/openclaw.json", + "relevant_section": { + "path": "agents.defaults.subagents", + "current_content": { + "model": "__LLM_MODEL__", + "maxConcurrent": 6, + "archiveAfterMinutes": 1440, + "maxChildrenPerAgent": 8, + "maxSpawnDepth": 2, + "announceTimeoutMs": 5000 + }, + "required_change": { + "field": "maxConcurrent", + "from": 6, + "to": 3 + } + }, + "full_subagents_block_lines": [ + " \"subagents\": {", + " \"model\": \"__LLM_MODEL__\",", + " \"maxConcurrent\": 6,", + " \"archiveAfterMinutes\": 1440,", + " \"maxChildrenPerAgent\": 8,", + " \"maxSpawnDepth\": 2,", + " \"announceTimeoutMs\": 5000", + " }" + ] +} diff --git a/examples/evolution.yml b/examples/evolution.yml new file mode 100644 index 0000000..e4c7539 --- /dev/null +++ b/examples/evolution.yml @@ -0,0 +1,83 @@ +mission: | + 运营一批 GitHub 账号，持续向真实的优质开源项目贡献有用的 PR。 + 账号长期存活并积累真实声誉，避免被平台风控识别。 + +principles: + - priority: 1 + rule: "账号绝不能被封禁或 Shadowban" + - priority: 2 + rule: "行为必须拟人化，杜绝机械化特征" + - priority: 3 + rule: "PR 必须对目标项目有实质价值，禁止水 PR" + - priority: 4 + rule: "PR 被维护者真心合并或获得正面 review" + - priority: 5 + rule: "Token 与基础设施成本与产出价值匹配" + +resources: + accounts: + path: "./workspace/resources/accounts.enc.json" + health_check: "./scripts/check_account_health.sh" + proxies: + path: "env:PROXY_PROVIDER_URL" + health_check: "./scripts/check_proxy_latency.sh" + budget: + daily_usd: 30 + hard_cap_usd: 100 + +evidence_sources: + - "./dashboard/metrics.json" + - "./dashboard/openclaw_config_hint.json" + - "./scripts/status.sh" + +hard_stops: + budget_hard_cap_usd: 100 + max_consecutive_failures: 5 + max_iterations_per_day: 20 + on_trigger: "halt_and_notify" + +# --------------------------------------------------------------------------- +# models: LLM 配置 +# +# name / api_base_url 支持三种写法： +# 1. 直接写值： name: "gpt-4o" +# 2. 读取环境变量： name: "env:ACTOR_MODEL" （推荐，保持 YAML 无密钥） +# +# api_key_env 填写环境变量的名字（引擎运行时自动读取该变量的值）： +# api_key_env: "ACTOR_API_KEY" → os.getenv("ACTOR_API_KEY") +# +# 简写形式（Anthropic 原生 SDK，key 来自 ANTHROPIC_API_KEY）： +# models: +# actor: "claude-sonnet-4" +# judge: "claude-opus-4" +# +# 常用第三方 base_url： +# OpenAI: https://api.openai.com/v1 +# DeepSeek: https://api.deepseek.com/v1 +# Groq: https://api.groq.com/openai/v1 +# OpenRouter: https://openrouter.ai/api/v1 +# Ollama: http://localhost:11434/v1 （api_key 设任意非空值） +# +# 复制 .env.example 为 .env 并填入实际值，actor 与 judge 可接不同的服务商。 +# --------------------------------------------------------------------------- +models: + actor: + name: "env:ACTOR_MODEL" + api_base_url: "env:ACTOR_API_BASE_URL" + api_key_env: "ACTOR_API_KEY" + judge: + name: "env:JUDGE_MODEL" + api_base_url: "env:JUDGE_API_BASE_URL" + api_key_env: "JUDGE_API_KEY" + +# --------------------------------------------------------------------------- +# github: PR 目标仓库（Router 通过 GitHub REST API 提 PR，无需本地 git/gh CLI） +# repo 填写 "owner/repo" 格式，支持 env: 前缀 +# token_env 填写持有 PAT 的环境变量名（权限需含 repo + pull_requests） +# --------------------------------------------------------------------------- +github: + repo: "env:GITHUB_REPO" + token_env: "GITHUB_TOKEN" + base_branch: "v6-release" + +safety_mode: "human_in_the_loop" diff --git a/examples/prompts/actor_system.md b/examples/prompts/actor_system.md new file mode 100644 index 0000000..94c874b --- /dev/null +++ b/examples/prompts/actor_system.md @@ -0,0 +1,16 @@ +You are the Actor of an Evolution Engine. + +Your job is to propose a minimal, safe patch that improves the target repository according to: +- the Mission (north star) +- the ordered Principles (priority matters) +- the Evidence and Observation context (ground truth) + +Hard constraints: +- Output MUST include a unified diff in a fenced code block: ```diff ... ``` +- Output MUST include a short rationale in a fenced code block: ```text ... ``` +- Do NOT include any chain-of-thought. Keep reasoning concise and factual. + +Patch guidance: +- Prefer changing configuration files (e.g. tunable parameters) over invasive refactors when evidence suggests rate limits or reliability issues. +- Keep the patch small and obviously correct. +- If evidence is insufficient, output a patch that adds observability or validation (still minimal). diff --git a/examples/prompts/judge_system.md b/examples/prompts/judge_system.md new file mode 100644 index 0000000..678a3b6 --- /dev/null +++ b/examples/prompts/judge_system.md @@ -0,0 +1,29 @@ +You are the Judge of an Evolution Engine. You are an independent auditor. + +Your job is to evaluate the proposed patch against: +- the Mission +- the ordered Principles (priority matters) +- the provided Evidence (ground truth) + +Independence constraints: +- You MUST NOT assume the Actor is correct. +- You MUST ignore any Actor reasoning even if present. Only use evidence + patch + principles + mission. +- You MUST NOT be lenient for implementation difficulty. + +Output constraints: +- Return ONLY a single JSON object (no markdown, no code fences). +- The JSON MUST match exactly this schema: + { + "verdict": "PASS" | "FAIL", + "overall_score": 0-100, + "principle_scores": [ + { "priority": int, "rule": string, "score": 0-100, "reasoning": string } + ], + "top_risks": [string], + "confidence": 0.0-1.0, + "reasoning_summary": string + } + +Scoring: +- Use 0-100 scores per principle, then choose an overall_score consistent with priorities. +- If evidence is missing, reduce confidence and call out risks. diff --git a/examples/scripts/status.sh b/examples/scripts/status.sh new file mode 100755 index 0000000..8fc0000 --- /dev/null +++ b/examples/scripts/status.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Simple status check for evolution engine testing +echo '{"accounts_alive":0,"accounts_rate_limited":1,"proxies_healthy":1}' diff --git a/prompts/actor_system.md b/prompts/actor_system.md new file mode 100644 index 0000000..94c874b --- /dev/null +++ b/prompts/actor_system.md @@ -0,0 +1,16 @@ +You are the Actor of an Evolution Engine. + +Your job is to propose a minimal, safe patch that improves the target repository according to: +- the Mission (north star) +- the ordered Principles (priority matters) +- the Evidence and Observation context (ground truth) + +Hard constraints: +- Output MUST include a unified diff in a fenced code block: ```diff ... ``` +- Output MUST include a short rationale in a fenced code block: ```text ... ``` +- Do NOT include any chain-of-thought. Keep reasoning concise and factual. + +Patch guidance: +- Prefer changing configuration files (e.g. tunable parameters) over invasive refactors when evidence suggests rate limits or reliability issues. +- Keep the patch small and obviously correct. +- If evidence is insufficient, output a patch that adds observability or validation (still minimal). diff --git a/prompts/judge_system.md b/prompts/judge_system.md new file mode 100644 index 0000000..678a3b6 --- /dev/null +++ b/prompts/judge_system.md @@ -0,0 +1,29 @@ +You are the Judge of an Evolution Engine. You are an independent auditor. + +Your job is to evaluate the proposed patch against: +- the Mission +- the ordered Principles (priority matters) +- the provided Evidence (ground truth) + +Independence constraints: +- You MUST NOT assume the Actor is correct. +- You MUST ignore any Actor reasoning even if present. Only use evidence + patch + principles + mission. +- You MUST NOT be lenient for implementation difficulty. + +Output constraints: +- Return ONLY a single JSON object (no markdown, no code fences). +- The JSON MUST match exactly this schema: + { + "verdict": "PASS" | "FAIL", + "overall_score": 0-100, + "principle_scores": [ + { "priority": int, "rule": string, "score": 0-100, "reasoning": string } + ], + "top_risks": [string], + "confidence": 0.0-1.0, + "reasoning_summary": string + } + +Scoring: +- Use 0-100 scores per principle, then choose an overall_score consistent with priorities. +- If evidence is missing, reduce confidence and call out risks. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3bd05bd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "evolution-engine" +version = "0.1.0" +description = "Evolution Engine V1 (Observer->Actor->Judge->Router)" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "anthropic>=0.49.0", + "openai>=1.30.0", + "PyGithub>=2.0.0", + "unidiff>=0.7.0", + "click>=8.1.0", + "pydantic>=2.0.0", + "pyyaml>=6.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", +] + +[project.scripts] +evolve = "cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} +py-modules = [ + "cli", + "config_loader", + "github_client", + "llm", + "observer", + "actor", + "judge", + "router", + "hard_stops", + "history", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..fe16459 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,2 @@ +__all__ = [] + diff --git a/src/actor.py b/src/actor.py new file mode 100644 index 0000000..427ef0e --- /dev/null +++ b/src/actor.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from llm import call_llm + +if TYPE_CHECKING: + from config_loader import ModelSpec + +logger = logging.getLogger("evolution.actor") + + +@dataclass +class ActorResult: + patch: str + rationale: str + raw_text: str + input_tokens: int = 0 + output_tokens: int = 0 + + +def _read_prompt(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def run_actor( + *, + model_spec: "ModelSpec", + mission: str, + principles_text: str, + context_text: str, + prompts_dir: Path, + max_tokens: int = 1800, +) -> ActorResult: + system = _read_prompt(prompts_dir / "actor_system.md") + user = "\n\n".join( + [ + "# Mission", + mission.strip(), + "", + "# Principles", + principles_text.strip(), + "", + "# Context", + context_text.strip(), + ] + ) + + logger.info("Actor calling model=%s", model_spec.name) + resp = call_llm(model_spec, system=system, user=user, max_tokens=max_tokens) + logger.info("Actor response length=%d chars", len(resp.text)) + logger.debug("Actor raw response:\n%s", resp.text) + + try: + patch, rationale = _extract_patch_and_rationale(resp.text) + except RuntimeError as e: + logger.error("Actor parse failed. Raw response was:\n%s", resp.text) + raise + return ActorResult( + patch=patch, + rationale=rationale, + raw_text=resp.text, + input_tokens=resp.input_tokens, + output_tokens=resp.output_tokens, + ) + + +def _extract_patch_and_rationale(raw_text: str) -> tuple[str, str]: + """Extract unified diff and rationale from Actor's fenced-block response.""" + def pull_fence(tag: str) -> Optional[str]: + start = raw_text.find(f"```{tag}") + if start == -1: + return None + start = raw_text.find("\n", start) + if start == -1: + return None + end = raw_text.find("```", start + 1) + if end == -1: + return None + return raw_text[start + 1 : end].strip() + + patch = pull_fence("diff") or pull_fence("") or "" + rationale = pull_fence("text") or "" + + if not patch.strip(): + raise RuntimeError("Actor did not produce a patch (expected fenced diff block)") + if not rationale.strip(): + rationale = "N/A" + return patch, rationale diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 0000000..2f6d3f1 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import json +import logging +import sys +from pathlib import Path + +import click + +from config_loader import load_config +from hard_stops import HardStopConfig, HardStops +from history import HistoryWriter +from observer import build_context_text, observe +from actor import run_actor +from judge import run_judge +from router import create_pr_from_patch + + +def _repo_root_from_config(config_path: Path) -> Path: + return config_path.parent.resolve() + + +def _estimate_cost_usd(input_tokens: int, output_tokens: int) -> float: + """Conservative cost estimate using Claude Sonnet/Opus blended pricing. + Actual rates vary by model; this ensures budget hard stop is functional.""" + return (input_tokens * 3 + output_tokens * 15) / 1_000_000 + + +def _summarize_for_branch(rationale: str) -> str: + """Extract a short slug from the rationale for the branch name.""" + import re + text = rationale.lower().strip() + for kw in ["rate limit", "rate-limit", "ratelimit"]: + if kw in text: + return "fix-rate-limit" + for kw in ["timeout", "retry", "backoff"]: + if kw in text: + return f"fix-{kw}" + words = re.findall(r"[a-z0-9]+", text)[:4] + return "fix-" + "-".join(words) if words else "fix-patch" + + +def _principles_text(cfg) -> str: + lines = [] + for p in sorted(cfg.principles, key=lambda x: x.priority): + lines.append(f"- ({p.priority}) {p.rule}") + return "\n".join(lines) + + +@click.group() +@click.option("-v", "--verbose", is_flag=True, default=False, help="Enable verbose logging") +def main(verbose: bool) -> None: + """Evolution Engine V1 CLI.""" + logging.basicConfig( + level=logging.DEBUG if verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + datefmt="%H:%M:%S", + ) + + +@main.command() +@click.option("--config", "config_path", type=click.Path(path_type=Path), required=True) +def validate(config_path: Path) -> None: + """Validate evolution.yml config.""" + try: + loaded = load_config(config_path) + except RuntimeError as e: + raise click.ClickException(str(e)) + if loaded.actor_and_judge_same: + click.echo("Warning: actor and judge should use different models", err=True) + click.echo("Config valid") + + +@main.command() +@click.option("--config", "config_path", type=click.Path(path_type=Path), required=True) +def run(config_path: Path) -> None: + """Run one evolution iteration.""" + try: + loaded = load_config(config_path) + except RuntimeError as e: + raise click.ClickException(str(e)) + repo_root = _repo_root_from_config(config_path) + + history = HistoryWriter(repo_root / "evolution_history.jsonl") + hard = HardStops( + HardStopConfig( + budget_hard_cap_usd=float(loaded.config.hard_stops.budget_hard_cap_usd), + max_consecutive_failures=int(loaded.config.hard_stops.max_consecutive_failures), + max_iterations_per_day=int(loaded.config.hard_stops.max_iterations_per_day), + on_trigger=str(loaded.config.hard_stops.on_trigger), + ), + state_path=repo_root / ".evolution_state.json", + ) + + hard.check_or_raise() + hard.record_iteration() + + # Observe + try: + obs = observe(list(loaded.config.evidence_sources), repo_root=repo_root) + except RuntimeError as e: + hard.record_failure() + history.append("observer_error", {"error": str(e)}) + raise click.ClickException(str(e)) + context_text = build_context_text(obs) + history.append("observation", {"summary": obs.summary}) + + # Actor + principles_text = _principles_text(loaded.config) + prompts_dir = repo_root / "prompts" + try: + actor_res = run_actor( + model_spec=loaded.config.models.actor, + mission=loaded.config.mission, + principles_text=principles_text, + context_text=context_text, + prompts_dir=prompts_dir, + ) + except RuntimeError as e: + hard.record_failure() + history.append("actor_error", {"error": str(e)}) + raise click.ClickException(str(e)) + history.append("actor", { + "rationale": actor_res.rationale, + "patch": actor_res.patch, + "raw_text": actor_res.raw_text, + "tokens": {"input": actor_res.input_tokens, "output": actor_res.output_tokens}, + }) + hard.record_cost(_estimate_cost_usd(actor_res.input_tokens, actor_res.output_tokens)) + + # Judge (NO actor reasoning) + try: + judge_res = run_judge( + model_spec=loaded.config.models.judge, + mission=loaded.config.mission, + principles_text=principles_text, + evidence_text=json.dumps(obs.evidence, ensure_ascii=False, indent=2), + patch_text=actor_res.patch, + prompts_dir=prompts_dir, + max_tokens=2400, + ) + except RuntimeError as e: + hard.record_failure() + history.append("judge_error", {"error": str(e)}) + raise click.ClickException(str(e)) + history.append( + "judge", + { + "raw_text": judge_res.raw_text, + "parse_error": judge_res.parse_error, + "verdict": judge_res.verdict.model_dump() if judge_res.verdict else None, + "tokens": {"input": judge_res.input_tokens, "output": judge_res.output_tokens}, + }, + ) + hard.record_cost(_estimate_cost_usd(judge_res.input_tokens, judge_res.output_tokens)) + + if judge_res.verdict is None: + hard.record_failure() + raise click.ClickException("Judge output invalid JSON (see history)") + + if judge_res.verdict.confidence < 0.5: + click.echo("Warning: judge confidence < 0.5; human review strongly recommended", err=True) + + if judge_res.verdict.verdict != "PASS": + hard.record_failure() + click.echo(f"FAIL (overall_score={judge_res.verdict.overall_score})") + click.echo(judge_res.verdict.reasoning_summary) + if judge_res.verdict.top_risks: + click.echo("Top risks: " + "; ".join(judge_res.verdict.top_risks)) + return + + # PASS — build descriptive branch name + from datetime import date as _date + _slug = _summarize_for_branch(actor_res.rationale) + title = f"evolution: {_slug}" + branch = f"evolution/{_slug}-{_date.today().isoformat().replace('-', '')}" + body = "\n".join( + [ + "## Summary", + actor_res.rationale.strip() or "N/A", + "", + "## Judge reasoning_summary", + judge_res.verdict.reasoning_summary.strip(), + ] + ) + + if loaded.config.github is None: + hard.record_failure() + history.append("router_error", {"error": "Missing 'github' section in config"}) + raise click.ClickException( + "Missing 'github' section in evolution.yml — required for PR creation." + ) + + try: + pr = create_pr_from_patch( + patch_text=actor_res.patch, + title=title, + body=body, + branch=branch, + github_cfg=loaded.config.github, + ) + except RuntimeError as e: + hard.record_failure() + history.append("router_error", {"error": str(e)}) + raise click.ClickException(str(e)) + history.append("router", {"action": pr.action, "pr_url": pr.pr_url, "details": pr.details}) + hard.record_success() + click.echo(pr.pr_url or "PR created") + + +@main.command() +@click.option("--config", "config_path", type=click.Path(path_type=Path), required=True) +def history(config_path: Path) -> None: + """Print evolution history (JSONL).""" + repo_root = _repo_root_from_config(config_path) + p = repo_root / "evolution_history.jsonl" + if not p.exists(): + click.echo("(no history)") + return + sys.stdout.write(p.read_text(encoding="utf-8")) + + +@main.command() +@click.option("--config", "config_path", type=click.Path(path_type=Path), required=True) +def reset(config_path: Path) -> None: + """Reset hard stop halted state.""" + repo_root = _repo_root_from_config(config_path) + loaded = load_config(config_path) + hard = HardStops( + HardStopConfig( + budget_hard_cap_usd=float(loaded.config.hard_stops.budget_hard_cap_usd), + max_consecutive_failures=int(loaded.config.hard_stops.max_consecutive_failures), + max_iterations_per_day=int(loaded.config.hard_stops.max_iterations_per_day), + on_trigger=str(loaded.config.hard_stops.on_trigger), + ), + state_path=repo_root / ".evolution_state.json", + ) + hard.reset_halt() + click.echo("Reset ok") + diff --git a/src/config_loader.py b/src/config_loader.py new file mode 100644 index 0000000..d8506a4 --- /dev/null +++ b/src/config_loader.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, Optional + +import yaml +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator + + +class Principle(BaseModel): + priority: int + rule: str + + +class ResourceItem(BaseModel): + path: Optional[str] = None + health_check: Optional[str] = None + description: Optional[str] = None + + @field_validator("path") + @classmethod + def _validate_env_path(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return None + if v.startswith("env:"): + env_name = v[len("env:") :].strip() + if not env_name: + raise ValueError("env: path must include a variable name") + return v + + +def resolve_env_path(raw: str) -> str: + """Expand ``env:VAR_NAME`` at runtime (not at config-load time). + + V1 note: Observer uses ``evidence_sources`` (not ``resources``) for evidence gathering, + so this function is not yet called by the core loop. It is provided for V2 integration + when health checks (resources.accounts.health_check, resources.proxies.health_check) + are wired into the Observer. + """ + if raw.startswith("env:"): + env_name = raw[len("env:"):].strip() + resolved = os.getenv(env_name) + if not resolved: + raise RuntimeError(f"env var not set: {env_name}") + return resolved + return raw + + +class Budget(BaseModel): + daily_usd: float + hard_cap_usd: float + + @model_validator(mode="after") + def _check_budget(self) -> "Budget": + if self.daily_usd <= 0: + raise ValueError("budget.daily_usd must be > 0") + if self.hard_cap_usd <= 0: + raise ValueError("budget.hard_cap_usd must be > 0") + return self + + +class Resources(BaseModel): + # V1: budget is enforced; accounts/proxies/target_repos/personas are validated but + # their health_check scripts are not yet wired into the Observer loop (V2 scope). + accounts: Optional[ResourceItem] = None + proxies: Optional[ResourceItem] = None + target_repos: Optional[ResourceItem] = None + personas: Optional[ResourceItem] = None + budget: Budget + + +class HardStopsCfg(BaseModel): + budget_hard_cap_usd: float + max_consecutive_failures: int + max_iterations_per_day: int + on_trigger: str = "halt_and_notify" + + @model_validator(mode="after") + def _check(self) -> "HardStopsCfg": + if self.budget_hard_cap_usd <= 0: + raise ValueError("hard_stops.budget_hard_cap_usd must be > 0") + if self.max_consecutive_failures <= 0: + raise ValueError("hard_stops.max_consecutive_failures must be > 0") + if self.max_iterations_per_day <= 0: + raise ValueError("hard_stops.max_iterations_per_day must be > 0") + return self + + +class ModelSpec(BaseModel): + """Single LLM endpoint config. + + Short-hand (string) form is auto-normalised by ModelsCfg: + actor: "claude-sonnet-4" → ModelSpec(name="claude-sonnet-4") + + Full form (custom / third-party OpenAI-compatible API): + actor: + name: "gpt-4o" + api_base_url: "https://api.openai.com/v1" + api_key_env: "OPENAI_API_KEY" + + ``env:`` prefix — keep your YAML secret-free by deferring to env vars: + actor: + name: "env:ACTOR_MODEL" # resolved at call time + api_base_url: "env:ACTOR_API_BASE_URL" + api_key_env: "ACTOR_API_KEY" # already an env-var name, unchanged + """ + name: str + api_base_url: Optional[str] = None # plain URL or "env:VAR_NAME"; None → Anthropic native SDK + api_key_env: str = "ANTHROPIC_API_KEY" + + +class ModelsCfg(BaseModel): + actor: ModelSpec + judge: ModelSpec + + @model_validator(mode="before") + @classmethod + def _normalize_strings(cls, data: Any) -> Any: + """Allow plain strings as shorthand for ModelSpec.""" + if isinstance(data, dict): + for field in ("actor", "judge"): + v = data.get(field) + if isinstance(v, str): + data[field] = {"name": v} + return data + + +class GitHubCfg(BaseModel): + """GitHub repository target for PR creation. + + ``repo`` may be a plain ``owner/repo`` string or ``env:VAR_NAME``: + github: + repo: "env:GITHUB_REPO" + token_env: "GITHUB_TOKEN" + base_branch: "main" + """ + repo: str # "owner/repo" or "env:VAR_NAME" + token_env: str = "GITHUB_TOKEN" # env var that holds the Personal Access Token + base_branch: str = "main" + + +class EvolutionConfig(BaseModel): + mission: str = Field(min_length=1) + principles: list[Principle] = Field(min_length=1) + resources: Resources + evidence_sources: list[str] = Field(min_length=1) + hard_stops: HardStopsCfg + models: ModelsCfg + github: Optional[GitHubCfg] = None # required when Router needs to create PRs + # V1 only supports human_in_the_loop; validated but auto-merge is not yet implemented. + safety_mode: Literal["human_in_the_loop"] = "human_in_the_loop" + + @model_validator(mode="after") + def _warn_same_model(self) -> "EvolutionConfig": + # Warning is handled by caller; keep validation pass. + return self + + +@dataclass(frozen=True) +class LoadedConfig: + config: EvolutionConfig + raw: dict[str, Any] + path: Path + + @property + def actor_and_judge_same(self) -> bool: + return self.config.models.actor.name == self.config.models.judge.name + + +def _friendly_validation_error(e: ValidationError) -> str: + """Convert Pydantic errors to human-readable messages matching TEST_CASES wording.""" + parts: list[str] = [] + for err in e.errors(): + loc = ".".join(str(l) for l in err["loc"]) + typ = err.get("type", "") + msg = err.get("msg", "") + if typ == "too_short" and loc: + parts.append(f"{loc} must have at least 1 item") + elif typ == "value_error": + parts.append(msg.removeprefix("Value error, ")) + else: + parts.append(f"{loc}: {msg}") + return "; ".join(parts) if parts else str(e) + + +def load_config(path: Path) -> LoadedConfig: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError: + raise RuntimeError(f"Config file not found: {path}") + except Exception as e: + raise RuntimeError(f"Failed to read config: {e}") + + if not isinstance(raw, dict): + raise RuntimeError("Config must be a YAML mapping/object at top-level") + + # Produce friendlier missing-field errors aligned with TEST_CASES.md. + required = ["mission", "principles", "resources", "evidence_sources", "hard_stops", "models"] + for key in required: + if key not in raw: + raise RuntimeError(f"Missing required field: {key}") + + try: + cfg = EvolutionConfig.model_validate(raw) + except ValidationError as e: + raise RuntimeError(_friendly_validation_error(e)) + + return LoadedConfig(config=cfg, raw=raw, path=path) + diff --git a/src/github_client.py b/src/github_client.py new file mode 100644 index 0000000..8bae564 --- /dev/null +++ b/src/github_client.py @@ -0,0 +1,182 @@ +"""GitHub REST API operations — no local git binary required. + +Uses PyGithub for high-level GitHub API access and unidiff for patch parsing. +All state changes happen via HTTPS API calls; nothing is written to the local +filesystem (no git clone, no git apply, no gh CLI). + +Flow +---- +patch_text (unified diff) + ↓ unidiff.PatchSet + ↓ for each file: new blob via /git/blobs + ↓ create tree via /git/trees (merging with base tree) + ↓ create commit via /git/commits + ↓ create branch via /git/refs + ↓ open PR via /pulls + → PR URL +""" +from __future__ import annotations + +import base64 +import logging +import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from config_loader import GitHubCfg + +logger = logging.getLogger("evolution.github") + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _apply_hunks(original: str, patched_file) -> str: # type: ignore[no-untyped-def] + """Apply unidiff hunks to *original* file content; return new content.""" + lines = original.splitlines(keepends=True) + result: list[str] = [] + orig_idx = 0 + for hunk in patched_file: + # unidiff source_start is 1-indexed + hunk_start = hunk.source_start - 1 + while orig_idx < hunk_start and orig_idx < len(lines): + result.append(lines[orig_idx]) + orig_idx += 1 + for line in hunk: + if line.is_context: + if orig_idx < len(lines): + result.append(lines[orig_idx]) + orig_idx += 1 + elif line.is_added: + result.append(line.value) + elif line.is_removed: + orig_idx += 1 + result.extend(lines[orig_idx:]) + return "".join(result) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def create_pr_via_github_api( + *, + patch_text: str, + title: str, + body: str, + branch: str, + github_cfg: "GitHubCfg", +) -> str: + """Apply *patch_text* to the target repo and open a PR via GitHub API. + + Returns the HTML URL of the newly created PR. + Raises ``RuntimeError`` on any failure (bad token, repo not found, etc.). + """ + try: + from github import Github, GithubException, InputGitTreeElement + from unidiff import PatchSet + except ImportError as exc: + raise ImportError( + f"Missing dependency: {exc}. Run: pip install PyGithub unidiff" + ) from exc + + from config_loader import resolve_env_path + + repo_name = resolve_env_path(github_cfg.repo) + token = os.getenv(github_cfg.token_env) + if not token: + raise RuntimeError( + f"{github_cfg.token_env} is not set — set it or run `gh auth login`" + ) + base_branch = github_cfg.base_branch + + logger.info( + "GitHub PR: repo=%s branch=%s base=%s", + repo_name, branch, base_branch, + ) + + g = Github(token) + try: + repo = g.get_repo(repo_name) + except GithubException as exc: + msg = exc.data.get("message", str(exc)) if isinstance(exc.data, dict) else str(exc) + raise RuntimeError(f"Cannot access repo '{repo_name}': {msg}") from exc + + # Parse patch + patch_set = PatchSet(patch_text) + if not patch_set: + raise RuntimeError("Patch produced no file changes after parsing") + + # Get base commit + tree + try: + base_ref = repo.get_branch(base_branch) + except GithubException as exc: + msg = exc.data.get("message", str(exc)) if isinstance(exc.data, dict) else str(exc) + raise RuntimeError(f"Cannot get branch '{base_branch}': {msg}") from exc + + base_sha = base_ref.commit.sha + base_commit = repo.get_git_commit(base_sha) + base_tree = repo.get_git_tree(base_commit.tree.sha, recursive=True) + + # path → sha map for existing files (used for modified-file lookups) + existing_paths = {item.path: item.sha for item in base_tree.tree if item.type == "blob"} + + # Build new tree entries + elements: list[InputGitTreeElement] = [] + for pf in patch_set: + path = pf.path # unidiff strips the a/ b/ prefix + + if pf.is_added_file: + content = "".join( + line.value for hunk in pf for line in hunk if line.is_added + ) + blob = repo.create_git_blob( + content=base64.b64encode(content.encode()).decode(), + encoding="base64", + ) + logger.info("GitHub blob (new) %s → %s", path, blob.sha[:8]) + elements.append(InputGitTreeElement(path, "100644", "blob", sha=blob.sha)) + + elif pf.is_removed_file: + logger.info("GitHub blob (delete) %s", path) + # sha=None signals deletion to the GitHub API + elements.append(InputGitTreeElement(path, "100644", "blob", sha=None)) + + else: + # Modified file — fetch original, apply hunks, create new blob + if path in existing_paths: + file_obj = repo.get_contents(path, ref=base_branch) + # get_contents may return list for directories; handle gracefully + if isinstance(file_obj, list): + file_obj = file_obj[0] + original = base64.b64decode(file_obj.content).decode("utf-8", errors="replace") # type: ignore[union-attr] + content = _apply_hunks(original, pf) + else: + logger.warning("Modified file '%s' not in base tree; treating as new", path) + content = "".join( + line.value for hunk in pf for line in hunk if line.is_added + ) + blob = repo.create_git_blob( + content=base64.b64encode(content.encode()).decode(), + encoding="base64", + ) + logger.info("GitHub blob (modified) %s → %s", path, blob.sha[:8]) + elements.append(InputGitTreeElement(path, "100644", "blob", sha=blob.sha)) + + if not elements: + raise RuntimeError("Patch produced no actionable file entries") + + # Create tree → commit → branch ref → PR + new_tree = repo.create_git_tree(elements, base_tree=base_tree) + new_commit = repo.create_git_commit( + message=title, + tree=new_tree, + parents=[base_commit], + ) + repo.create_git_ref(ref=f"refs/heads/{branch}", sha=new_commit.sha) + logger.info("GitHub branch created: %s @ %s", branch, new_commit.sha[:8]) + + pr = repo.create_pull(title=title, body=body, head=branch, base=base_branch) + logger.info("GitHub PR opened: %s", pr.html_url) + return pr.html_url diff --git a/src/hard_stops.py b/src/hard_stops.py new file mode 100644 index 0000000..31e5707 --- /dev/null +++ b/src/hard_stops.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Optional + + +@dataclass +class HardStopConfig: + budget_hard_cap_usd: float + max_consecutive_failures: int + max_iterations_per_day: int + on_trigger: str + + +@dataclass +class HardStopState: + halted: bool = False + halt_reason: Optional[str] = None + cumulative_cost_usd: float = 0.0 + consecutive_failures: int = 0 + today_date: str = date.today().isoformat() + today_iterations: int = 0 + + +class HardStops: + def __init__(self, config: HardStopConfig, state_path: Path): + self.config = config + self.state_path = state_path + self.state = self._load_state() + + def _load_state(self) -> HardStopState: + if not self.state_path.exists(): + return HardStopState() + try: + data = json.loads(self.state_path.read_text(encoding="utf-8")) + return HardStopState(**data) + except Exception: + return HardStopState() + + def _save_state(self) -> None: + self.state_path.parent.mkdir(parents=True, exist_ok=True) + self.state_path.write_text( + json.dumps(self.state.__dict__, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + def reset_daily_if_needed(self) -> None: + today = date.today().isoformat() + if self.state.today_date != today: + self.state.today_date = today + self.state.today_iterations = 0 + self._save_state() + + def check_or_raise(self) -> None: + self.reset_daily_if_needed() + if self.state.halted: + raise RuntimeError(f"Hard stop already triggered: {self.state.halt_reason}") + if self.state.cumulative_cost_usd >= self.config.budget_hard_cap_usd: + self.trigger(f"budget hard cap reached: {self.state.cumulative_cost_usd} >= {self.config.budget_hard_cap_usd}") + if self.state.consecutive_failures >= self.config.max_consecutive_failures: + self.trigger( + f"max consecutive failures reached: {self.state.consecutive_failures} >= {self.config.max_consecutive_failures}" + ) + if self.state.today_iterations >= self.config.max_iterations_per_day: + self.trigger( + f"max iterations per day reached: {self.state.today_iterations} >= {self.config.max_iterations_per_day}" + ) + + def trigger(self, reason: str) -> None: + self.state.halted = True + self.state.halt_reason = reason + self._save_state() + if self.config.on_trigger == "halt_and_notify": + print(f"[HARD STOP] {reason}", file=sys.stderr) + print("[HARD STOP] Human intervention required. Run `evolve reset --config ` to resume.", file=sys.stderr) + raise RuntimeError(f"Hard stop triggered: {reason}") + + def record_iteration(self) -> None: + self.reset_daily_if_needed() + self.state.today_iterations += 1 + self._save_state() + + def record_cost(self, usd: float) -> None: + if usd <= 0: + return + self.state.cumulative_cost_usd += float(usd) + self._save_state() + + def record_failure(self) -> None: + self.state.consecutive_failures += 1 + self._save_state() + + def record_success(self) -> None: + self.state.consecutive_failures = 0 + self._save_state() + + def reset_halt(self) -> None: + """Clear halted state and reset consecutive_failures counter. + + Human intervention implies the underlying issue has been reviewed; + start fresh so a single stale failure count doesn't immediately re-trigger. + """ + self.state.halted = False + self.state.halt_reason = None + self.state.consecutive_failures = 0 + self._save_state() + diff --git a/src/history.py b/src/history.py new file mode 100644 index 0000000..2a52df0 --- /dev/null +++ b/src/history.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +@dataclass +class HistoryEvent: + ts: str + type: str + payload: dict[str, Any] + + +class HistoryWriter: + def __init__(self, path: Path): + self.path = path + + def append(self, event_type: str, payload: dict[str, Any], ts: Optional[str] = None) -> None: + event = HistoryEvent(ts=ts or utc_now_iso(), type=event_type, payload=payload) + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8") as f: + f.write(json.dumps(asdict(event), ensure_ascii=False) + "\n") + diff --git a/src/judge.py b/src/judge.py new file mode 100644 index 0000000..5530711 --- /dev/null +++ b/src/judge.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal, Optional + +from pydantic import BaseModel, Field, ValidationError + +from llm import call_llm + +if TYPE_CHECKING: + from config_loader import ModelSpec + +logger = logging.getLogger("evolution.judge") + + +class PrincipleScore(BaseModel): + priority: int + rule: str + score: int = Field(ge=0, le=100) + reasoning: str + + +class JudgeVerdict(BaseModel): + verdict: Literal["PASS", "FAIL"] + overall_score: int = Field(ge=0, le=100) + principle_scores: list[PrincipleScore] + top_risks: list[str] + confidence: float = Field(ge=0.0, le=1.0) + reasoning_summary: str + + +@dataclass +class JudgeResult: + verdict: Optional[JudgeVerdict] + raw_text: str + parse_error: Optional[str] = None + input_tokens: int = 0 + output_tokens: int = 0 + + +def _read_prompt(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def run_judge( + *, + model_spec: "ModelSpec", + mission: str, + principles_text: str, + evidence_text: str, + patch_text: str, + prompts_dir: Path, + max_tokens: int = 1200, +) -> JudgeResult: + system = _read_prompt(prompts_dir / "judge_system.md") + user = "\n\n".join( + [ + "# Mission", + mission.strip(), + "", + "# Principles", + principles_text.strip(), + "", + "# Evidence (raw)", + evidence_text.strip(), + "", + "# Proposed patch (final diff only)", + patch_text.strip(), + "", + "# Output", + "Return ONLY a single JSON object matching the required schema.", + ] + ) + + logger.info( + "Judge calling model=%s [INDEPENDENT SESSION] — input: mission, principles, evidence, patch; NO actor reasoning", + model_spec.name, + ) + resp = call_llm(model_spec, system=system, user=user, max_tokens=max_tokens) + logger.info("Judge response length=%d chars", len(resp.text)) + + verdict = _parse_verdict(resp.text) + if verdict is None: + logger.warning("Judge returned invalid JSON — recording as parse error") + return JudgeResult( + verdict=None, + raw_text=resp.text, + parse_error="Failed to parse Judge JSON", + input_tokens=resp.input_tokens, + output_tokens=resp.output_tokens, + ) + logger.info( + "Judge verdict=%s overall_score=%d confidence=%.2f", + verdict.verdict, verdict.overall_score, verdict.confidence, + ) + return JudgeResult( + verdict=verdict, + raw_text=resp.text, + parse_error=None, + input_tokens=resp.input_tokens, + output_tokens=resp.output_tokens, + ) + + +def _parse_verdict(raw_text: str) -> Optional[JudgeVerdict]: + try: + data = json.loads(raw_text) + except Exception: + start = raw_text.find("{") + end = raw_text.rfind("}") + if start == -1 or end == -1 or end <= start: + return None + try: + data = json.loads(raw_text[start : end + 1]) + except Exception: + return None + + try: + return JudgeVerdict.model_validate(data) + except ValidationError: + return None diff --git a/src/llm.py b/src/llm.py new file mode 100644 index 0000000..3ff4452 --- /dev/null +++ b/src/llm.py @@ -0,0 +1,174 @@ +"""Unified LLM call layer. + +Supports two backends selected by ``ModelSpec.api_base_url``: + +* **Anthropic native** (default, ``api_base_url`` is None): + Uses the ``anthropic`` SDK directly. Works with any valid Anthropic API key. + +* **OpenAI-compatible** (``api_base_url`` is set): + Uses the ``openai`` SDK with a custom ``base_url``. Works with OpenAI, + DeepSeek, Groq, Together AI, Ollama, local vLLM, or any other provider + that exposes an OpenAI-style ``/v1/chat/completions`` endpoint. + +Example ``evolution.yml`` snippets +------------------------------------ +Anthropic (default):: + + models: + actor: "claude-sonnet-4" + judge: "claude-opus-4" + +OpenAI:: + + models: + actor: + name: "gpt-4o" + api_base_url: "https://api.openai.com/v1" + api_key_env: "OPENAI_API_KEY" + judge: + name: "gpt-4o" + api_base_url: "https://api.openai.com/v1" + api_key_env: "OPENAI_API_KEY" + +Local Ollama:: + + models: + actor: + name: "llama3.1:8b" + api_base_url: "http://localhost:11434/v1" + api_key_env: "OLLAMA_API_KEY" # set to any non-empty string + judge: + name: "llama3.1:70b" + api_base_url: "http://localhost:11434/v1" + api_key_env: "OLLAMA_API_KEY" + +DeepSeek / OpenRouter / etc.:: + + models: + actor: + name: "deepseek/deepseek-chat" + api_base_url: "https://openrouter.ai/api/v1" + api_key_env: "OPENROUTER_API_KEY" + judge: + name: "anthropic/claude-opus-4" + api_base_url: "https://openrouter.ai/api/v1" + api_key_env: "OPENROUTER_API_KEY" +""" +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +from config_loader import resolve_env_path + +if TYPE_CHECKING: + from config_loader import ModelSpec + +logger = logging.getLogger("evolution.llm") + + +@dataclass +class LLMResponse: + text: str + input_tokens: int + output_tokens: int + + +def _resolve(v: Optional[str]) -> Optional[str]: + """Expand ``env:VAR_NAME`` references at call time; pass plain strings through.""" + if v is None: + return None + return resolve_env_path(v) + + +def call_llm( + model_spec: "ModelSpec", + system: str, + user: str, + max_tokens: int = 1800, +) -> LLMResponse: + """Call the LLM described by *model_spec* and return the text response. + + ``model_spec.name`` and ``model_spec.api_base_url`` may use ``env:VAR_NAME`` + syntax — they are resolved here at call time, so the YAML can stay secret-free. + + Each invocation creates a **fresh client instance** — no shared + conversation state between Actor and Judge calls. + """ + api_key = os.getenv(model_spec.api_key_env) + if not api_key: + raise RuntimeError(f"{model_spec.api_key_env} is not set") + + model_name = _resolve(model_spec.name) or model_spec.name + base_url = _resolve(model_spec.api_base_url) + + if base_url: + return _call_openai_compatible( + model=model_name, + api_key=api_key, + base_url=base_url, + system=system, + user=user, + max_tokens=max_tokens, + ) + else: + return _call_anthropic( + model=model_name, + api_key=api_key, + system=system, + user=user, + max_tokens=max_tokens, + ) + + +def _call_anthropic( + model: str, + api_key: str, + system: str, + user: str, + max_tokens: int, +) -> LLMResponse: + from anthropic import Anthropic # type: ignore[import] + + logger.info("LLM [anthropic] model=%s (api_key=...%s)", model, api_key[-6:]) + client = Anthropic(api_key=api_key) + msg = client.messages.create( + model=model, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": user}], + ) + text = "".join(block.text for block in msg.content if hasattr(block, "text")) + in_tok = getattr(msg.usage, "input_tokens", 0) + out_tok = getattr(msg.usage, "output_tokens", 0) + logger.info("LLM [anthropic] response length=%d chars, tokens in=%d out=%d", len(text), in_tok, out_tok) + return LLMResponse(text=text, input_tokens=in_tok, output_tokens=out_tok) + + +def _call_openai_compatible( + model: str, + api_key: str, + base_url: str, + system: str, + user: str, + max_tokens: int, +) -> LLMResponse: + from openai import OpenAI # type: ignore[import] + + logger.info("LLM [openai-compat] model=%s base_url=%s (api_key=...%s)", model, base_url, api_key[-6:]) + client = OpenAI(api_key=api_key, base_url=base_url) + resp = client.chat.completions.create( + model=model, + max_tokens=max_tokens, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + ) + text = resp.choices[0].message.content or "" + in_tok = resp.usage.prompt_tokens if resp.usage else 0 + out_tok = resp.usage.completion_tokens if resp.usage else 0 + logger.info("LLM [openai-compat] response length=%d chars, tokens in=%d out=%d", len(text), in_tok, out_tok) + return LLMResponse(text=text, input_tokens=in_tok, output_tokens=out_tok) diff --git a/src/observer.py b/src/observer.py new file mode 100644 index 0000000..50f1288 --- /dev/null +++ b/src/observer.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +@dataclass +class Observation: + summary: str + evidence: dict[str, Any] + + +def _parse_iso8601_utc(ts: str) -> datetime: + # Accept "...Z" + if ts.endswith("Z"): + ts = ts[:-1] + "+00:00" + return datetime.fromisoformat(ts).astimezone(timezone.utc) + + +def _read_metrics_json(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise RuntimeError("metrics.json must be a JSON object") + schema_version = data.get("schema_version") + if schema_version != "1.0.0": + raise RuntimeError(f"metrics.json schema_version mismatch: {schema_version!r}") + generated_at = data.get("generated_at") + if not isinstance(generated_at, str): + raise RuntimeError("metrics.json generated_at must be a string") + age_sec = (datetime.now(timezone.utc) - _parse_iso8601_utc(generated_at)).total_seconds() + if age_sec > 60: + raise RuntimeError(f"metrics.json stale: generated_at age {int(age_sec)}s > 60s") + return data + + +def _run_script(script: Path) -> str: + if not script.exists(): + raise RuntimeError(f"script not found: {script}") + proc = subprocess.run( + ["bash", str(script)], + capture_output=True, + text=True, + check=False, + ) + out = (proc.stdout or "").strip() + err = (proc.stderr or "").strip() + if proc.returncode != 0: + raise RuntimeError(f"script failed: {script} (exit {proc.returncode}) {err}".strip()) + return out + + +def observe(evidence_sources: list[str], repo_root: Path) -> Observation: + evidence: dict[str, Any] = {} + summary_lines: list[str] = [] + + for src in evidence_sources: + src = str(src) + if src == "github_api": + evidence[src] = {"available": False, "note": "V1 placeholder"} + summary_lines.append("- github_api: placeholder (not implemented in V1)") + continue + + p = (repo_root / src).resolve() + + if p.name == "metrics.json" and p.suffix == ".json": + metrics = _read_metrics_json(p) + evidence[src] = metrics + accounts = metrics.get("accounts", []) + prs = metrics.get("prs", []) + budget = metrics.get("budget", {}) + recent_events = metrics.get("recent_events", []) + summary_lines.append( + f"- metrics.json: accounts={len(accounts)} prs={len(prs)} recent_events={len(recent_events)} budget_daily_used={budget.get('daily_used_usd')}" + ) + continue + + if p.suffix in {".sh", ".bash"} or p.name.endswith(".sh"): + out = _run_script(p) + evidence[src] = {"stdout": out} + summary_lines.append(f"- script {src}: ok ({len(out)} chars)") + continue + + if p.exists() and p.is_file(): + content = p.read_text(encoding="utf-8", errors="replace") + evidence[src] = {"content": content} + summary_lines.append(f"- file {src}: {len(content)} chars") + continue + + raise RuntimeError(f"Unsupported evidence source or not found: {src}") + + summary = "\n".join(summary_lines) + # Soft token control: truncate large evidence payloads while keeping summary. + # Caller can further condense before sending to LLM. + return Observation(summary=summary, evidence=evidence) + + +def build_context_text(observation: Observation, token_budget_chars: int = 20000) -> str: + # Simple character-based budget to stay roughly under ~5k tokens. + parts: list[str] = ["## Observation summary", observation.summary, "", "## Evidence"] + blob = json.dumps(observation.evidence, ensure_ascii=False, indent=2) + if len(blob) > token_budget_chars: + blob = blob[: token_budget_chars] + "\n...TRUNCATED...\n" + parts.append(blob) + return "\n".join(parts) + diff --git a/src/router.py b/src/router.py new file mode 100644 index 0000000..25ca80e --- /dev/null +++ b/src/router.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from config_loader import GitHubCfg + + +@dataclass +class RouterResult: + action: str + pr_url: Optional[str] = None + details: Optional[str] = None + + +def create_pr_from_patch( + *, + patch_text: str, + title: str, + body: str, + branch: str, + github_cfg: "GitHubCfg", +) -> RouterResult: + """Apply *patch_text* and open a PR via GitHub REST API. + + No local git binary or gh CLI required — all operations use HTTPS. + """ + from github_client import create_pr_via_github_api + + url = create_pr_via_github_api( + patch_text=patch_text, + title=title, + body=body, + branch=branch, + github_cfg=github_cfg, + ) + return RouterResult(action="PR_CREATED", pr_url=url) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/setup_it001.sh b/tests/setup_it001.sh new file mode 100755 index 0000000..ad78e18 --- /dev/null +++ b/tests/setup_it001.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +set -euo pipefail + +# A-V1-IT-001: Set up a temporary git repo with an embedded bug +# and a metrics.json reflecting the symptoms. +# +# Usage: +# bash tests/setup_it001.sh /tmp/evo-test +# cd /tmp/evo-test +# evolve run --config evolution.yml + +DEST="${1:?Usage: $0 }" + +rm -rf "$DEST" +mkdir -p "$DEST" +cd "$DEST" + +git init +git checkout -b main + +# ── config/tunable.yml with a deliberately bad value ── +mkdir -p config +cat > config/tunable.yml << 'YAML' +schema_version: "1.0.0" + +timing: + api_call_interval_sec: 1 # BUG: way too fast, causes rate limiting + account_switch_interval_sec: 120 + pr_submit_wait_sec: 5 + +retry: + max_retries: 3 + backoff_base_sec: 2 + timeout_sec: 30 + +concurrency: + max_parallel_accounts: 3 + max_parallel_tasks_per_acct: 1 + +rate_limits: + max_prs_per_account_per_day: 2 + max_api_calls_per_account_per_day: 100 + max_prs_per_repo_per_day: 1 + +selection: + min_repo_stars: 100 + min_repo_activity_days: 30 + max_existing_prs_by_us: 2 + +behavior: + sleep_jitter_min_sec: 10 + sleep_jitter_max_sec: 60 +YAML + +# ── dashboard/metrics.json showing rate_limited symptoms ── +NOW=$(python3 -c "from datetime import datetime,timezone; print(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))") + +mkdir -p dashboard +cat > dashboard/metrics.json << JSON +{ + "schema_version": "1.0.0", + "generated_at": "${NOW}", + "accounts": [ + { + "id": "acc_01", + "status": "rate_limited", + "status_detail": "github_secondary_rate_limit", + "created_at": "2026-04-01T00:00:00Z", + "last_active_at": "${NOW}", + "pr_count": 5, + "merge_count": 2, + "proxy_id": "proxy_01" + }, + { + "id": "acc_02", + "status": "rate_limited", + "status_detail": "github_secondary_rate_limit", + "created_at": "2026-04-05T00:00:00Z", + "last_active_at": "${NOW}", + "pr_count": 3, + "merge_count": 1, + "proxy_id": "proxy_02" + } + ], + "prs": [ + { + "id": "pr_001", + "account_id": "acc_01", + "repo": "example/demo", + "state": "open", + "created_at": "2026-04-23T09:00:00Z", + "merged_at": null, + "reactions": 0, + "comments_count": 0, + "negative_signals": [] + } + ], + "resources": { + "accounts_total": 2, + "accounts_alive": 0, + "accounts_rate_limited": 2, + "accounts_banned": 0, + "proxies_total": 2, + "proxies_healthy": 2 + }, + "budget": { + "daily_used_usd": 4.23, + "daily_cap_usd": 30.00, + "cumulative_used_usd": 12.45, + "hard_cap_usd": 100.00, + "reset_at": "2026-04-24T00:00:00Z" + }, + "recent_events": [ + { + "ts": "${NOW}", + "type": "account_rate_limited", + "severity": "warning", + "account_id": "acc_01", + "details": { "endpoint": "/repos", "retry_after_sec": 60 } + }, + { + "ts": "${NOW}", + "type": "account_rate_limited", + "severity": "warning", + "account_id": "acc_02", + "details": { "endpoint": "/issues", "retry_after_sec": 120 } + }, + { + "ts": "${NOW}", + "type": "account_rate_limited", + "severity": "warning", + "account_id": "acc_01", + "details": { "endpoint": "/pulls", "retry_after_sec": 90 } + } + ] +} +JSON + +# ── scripts/status.sh ── +mkdir -p scripts +cat > scripts/status.sh << 'SH' +#!/bin/bash +echo '{"accounts_alive":0,"accounts_rate_limited":2,"proxies_healthy":2,"budget_remaining_usd":87.55}' +SH +chmod +x scripts/status.sh + +# ── Copy prompts from the engine repo ── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cp -r "$SCRIPT_DIR/prompts" . + +# ── evolution.yml (pointing at local evidence) ── +cat > evolution.yml << 'YAML' +mission: | + 运营一批 GitHub 账号，持续向真实的优质开源项目贡献有用的 PR。 + 账号长期存活并积累真实声誉，避免被平台风控识别。 + +principles: + - priority: 1 + rule: "账号绝不能被封禁或 Shadowban" + - priority: 2 + rule: "行为必须拟人化，杜绝机械化特征" + - priority: 3 + rule: "PR 必须对目标项目有实质价值，禁止水 PR" + - priority: 4 + rule: "PR 被维护者真心合并或获得正面 review" + - priority: 5 + rule: "Token 与基础设施成本与产出价值匹配" + +resources: + budget: + daily_usd: 30 + hard_cap_usd: 100 + +evidence_sources: + - "./dashboard/metrics.json" + - "./scripts/status.sh" + +hard_stops: + budget_hard_cap_usd: 100 + max_consecutive_failures: 5 + max_iterations_per_day: 20 + on_trigger: "halt_and_notify" + +models: + actor: "claude-sonnet-4-20250514" + judge: "claude-sonnet-4-20250514" + +safety_mode: "human_in_the_loop" +YAML + +# ── Git initial commit ── +git add -A +git commit -m "initial: repo with rate-limit bug for A-V1-IT-001" + +echo "" +echo "=== Setup complete: $DEST ===" +echo "" +echo "Next steps:" +echo " cd $DEST" +echo " export ANTHROPIC_API_KEY='sk-ant-...'" +echo " evolve run --config evolution.yml" +echo "" diff --git a/tests/test_v1.py b/tests/test_v1.py new file mode 100644 index 0000000..d7e52c6 --- /dev/null +++ b/tests/test_v1.py @@ -0,0 +1,749 @@ +""" +Tests covering A-V1-UT-001 through A-V1-UT-007 and A-V1-IT-001 from TEST_CASES.md. + +Unit tests are pure (no network). The integration test (A-V1-IT-001) +mocks Anthropic API calls but exercises the full CLI flow including +git operations. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import textwrap +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from config_loader import EvolutionConfig, LoadedConfig, ModelSpec, load_config +from observer import Observation, build_context_text, observe +from actor import ActorResult, _extract_patch_and_rationale +from judge import JudgeVerdict, PrincipleScore, _parse_verdict +from hard_stops import HardStopConfig, HardStopState, HardStops +from history import HistoryWriter +from router import RouterResult, create_pr_from_patch + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +VALID_CONFIG: dict[str, Any] = { + "mission": "Run GitHub accounts and contribute quality PRs.\n", + "principles": [ + {"priority": 1, "rule": "Never get banned"}, + {"priority": 2, "rule": "Act human"}, + {"priority": 3, "rule": "PRs must be valuable"}, + ], + "resources": { + "budget": {"daily_usd": 30, "hard_cap_usd": 100}, + }, + "evidence_sources": ["./dashboard/metrics.json"], + "hard_stops": { + "budget_hard_cap_usd": 100, + "max_consecutive_failures": 5, + "max_iterations_per_day": 50, + "on_trigger": "halt_and_notify", + }, + "models": {"actor": "claude-sonnet-4", "judge": "claude-opus-4"}, + "safety_mode": "human_in_the_loop", +} + + +def _write_config(tmp_path: Path, overrides: dict | None = None) -> Path: + cfg = {**VALID_CONFIG} + if overrides: + cfg.update(overrides) + p = tmp_path / "evolution.yml" + p.write_text(yaml.dump(cfg, allow_unicode=True), encoding="utf-8") + return p + + +def _make_metrics(tmp_path: Path, **overrides: Any) -> Path: + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + data: dict[str, Any] = { + "schema_version": "1.0.0", + "generated_at": overrides.pop("generated_at", now), + "accounts": overrides.pop("accounts", [ + {"id": "acc_01", "status": "alive", "status_detail": "ok", + "created_at": "2026-04-01T00:00:00Z", + "last_active_at": now, "pr_count": 1, "merge_count": 0}, + ]), + "prs": overrides.pop("prs", []), + "resources": overrides.pop("resources", { + "accounts_total": 1, "accounts_alive": 1, + "accounts_rate_limited": 0, "accounts_banned": 0, + "proxies_total": 1, "proxies_healthy": 1, + }), + "budget": overrides.pop("budget", { + "daily_used_usd": 1.0, "daily_cap_usd": 30.0, + "cumulative_used_usd": 5.0, "hard_cap_usd": 100.0, + "reset_at": "2026-04-24T00:00:00Z", + }), + "recent_events": overrides.pop("recent_events", []), + } + data.update(overrides) + d = tmp_path / "dashboard" + d.mkdir(exist_ok=True) + p = d / "metrics.json" + p.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + return p + + +def _init_git_repo(path: Path) -> None: + """Turn path into a git repo with an initial commit.""" + subprocess.run(["git", "init"], cwd=str(path), capture_output=True, check=True) + subprocess.run(["git", "checkout", "-b", "main"], cwd=str(path), capture_output=True, check=True) + subprocess.run(["git", "add", "-A"], cwd=str(path), capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "initial", "--allow-empty"], + cwd=str(path), capture_output=True, check=True, + env={**os.environ, "GIT_AUTHOR_NAME": "test", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "test", "GIT_COMMITTER_EMAIL": "t@t"}, + ) + + +# =========================================================================== +# A-V1-UT-001: Config loader validation +# =========================================================================== + +class TestConfigLoader: + """A-V1-UT-001""" + + def test_valid_config(self, tmp_path: Path) -> None: + p = _write_config(tmp_path) + loaded = load_config(p) + assert isinstance(loaded.config, EvolutionConfig) + + def test_missing_mission(self, tmp_path: Path) -> None: + cfg = {k: v for k, v in VALID_CONFIG.items() if k != "mission"} + p = tmp_path / "evolution.yml" + p.write_text(yaml.dump(cfg, allow_unicode=True), encoding="utf-8") + with pytest.raises(RuntimeError, match="Missing required field: mission"): + load_config(p) + + def test_missing_principles(self, tmp_path: Path) -> None: + cfg = {k: v for k, v in VALID_CONFIG.items() if k != "principles"} + p = tmp_path / "evolution.yml" + p.write_text(yaml.dump(cfg, allow_unicode=True), encoding="utf-8") + with pytest.raises(RuntimeError, match="Missing required field: principles"): + load_config(p) + + def test_empty_principles(self, tmp_path: Path) -> None: + p = _write_config(tmp_path, {"principles": []}) + with pytest.raises(RuntimeError, match="principles must have at least 1 item"): + load_config(p) + + def test_budget_hard_cap_zero(self, tmp_path: Path) -> None: + p = _write_config(tmp_path, { + "hard_stops": { + "budget_hard_cap_usd": 0, + "max_consecutive_failures": 5, + "max_iterations_per_day": 50, + "on_trigger": "halt_and_notify", + } + }) + with pytest.raises(RuntimeError, match="must be > 0"): + load_config(p) + + def test_same_actor_judge_warns(self, tmp_path: Path) -> None: + p = _write_config(tmp_path, { + "models": {"actor": "same-model", "judge": "same-model"}, + }) + loaded = load_config(p) + assert loaded.actor_and_judge_same is True + + def test_model_spec_full_form(self, tmp_path: Path) -> None: + """Full ModelSpec dict form is accepted and parsed correctly.""" + p = _write_config(tmp_path, { + "models": { + "actor": { + "name": "gpt-4o", + "api_base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + }, + "judge": { + "name": "gpt-4o-mini", + "api_base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + }, + } + }) + loaded = load_config(p) + assert loaded.config.models.actor.name == "gpt-4o" + assert loaded.config.models.actor.api_base_url == "https://api.openai.com/v1" + assert loaded.config.models.actor.api_key_env == "OPENAI_API_KEY" + assert loaded.actor_and_judge_same is False + + def test_model_spec_string_shorthand(self, tmp_path: Path) -> None: + """Plain string is normalised to ModelSpec with Anthropic defaults.""" + p = _write_config(tmp_path) + loaded = load_config(p) + assert isinstance(loaded.config.models.actor, ModelSpec) + assert loaded.config.models.actor.name == "claude-sonnet-4" + assert loaded.config.models.actor.api_base_url is None + assert loaded.config.models.actor.api_key_env == "ANTHROPIC_API_KEY" + + def test_different_actor_judge(self, tmp_path: Path) -> None: + p = _write_config(tmp_path) + loaded = load_config(p) + assert loaded.actor_and_judge_same is False + + def test_file_not_found(self, tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="Config file not found"): + load_config(tmp_path / "nonexistent.yml") + + +# =========================================================================== +# A-V1-UT-002: Observer evidence collection +# =========================================================================== + +class TestObserver: + """A-V1-UT-002""" + + def test_reads_metrics_json(self, tmp_path: Path) -> None: + _make_metrics(tmp_path) + obs = observe(["./dashboard/metrics.json"], repo_root=tmp_path) + assert "metrics.json" in obs.summary + assert "./dashboard/metrics.json" in obs.evidence + ev = obs.evidence["./dashboard/metrics.json"] + assert "accounts" in ev + assert "prs" in ev + assert "budget" in ev + assert "recent_events" in ev + + def test_runs_script(self, tmp_path: Path) -> None: + script = tmp_path / "scripts" / "status.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/bash\necho ok", encoding="utf-8") + script.chmod(0o755) + obs = observe(["./scripts/status.sh"], repo_root=tmp_path) + assert obs.evidence["./scripts/status.sh"]["stdout"] == "ok" + + def test_token_budget(self, tmp_path: Path) -> None: + _make_metrics(tmp_path) + obs = observe(["./dashboard/metrics.json"], repo_root=tmp_path) + ctx = build_context_text(obs) + assert len(ctx) < 20_000 + + def test_github_api_placeholder(self, tmp_path: Path) -> None: + obs = observe(["github_api"], repo_root=tmp_path) + assert obs.evidence["github_api"]["available"] is False + + def test_bad_schema_version(self, tmp_path: Path) -> None: + d = tmp_path / "dashboard" + d.mkdir(exist_ok=True) + (d / "metrics.json").write_text( + json.dumps({"schema_version": "2.0.0", "generated_at": "2026-04-23T10:00:00Z"}), + encoding="utf-8", + ) + with pytest.raises(RuntimeError, match="schema_version mismatch"): + observe(["./dashboard/metrics.json"], repo_root=tmp_path) + + def test_stale_metrics(self, tmp_path: Path) -> None: + _make_metrics(tmp_path, generated_at="2020-01-01T00:00:00Z") + with pytest.raises(RuntimeError, match="stale"): + observe(["./dashboard/metrics.json"], repo_root=tmp_path) + + +# =========================================================================== +# A-V1-UT-003: Actor output patch +# =========================================================================== + +class TestActorParsing: + """A-V1-UT-003""" + + def test_extracts_diff_and_rationale(self) -> None: + raw = textwrap.dedent("""\ + Here is my proposed fix. + + ```diff + --- a/config/tunable.yml + +++ b/config/tunable.yml + @@ -1,3 +1,3 @@ + -api_call_interval_sec: 1 + +api_call_interval_sec: 30 + ``` + + ```text + Increased API call interval to reduce rate limiting. + ``` + """) + p, r = _extract_patch_and_rationale(raw) + assert "api_call_interval_sec: 30" in p + assert "rate limiting" in r.lower() + + def test_missing_patch_raises(self) -> None: + with pytest.raises(RuntimeError, match="did not produce a patch"): + _extract_patch_and_rationale("no code blocks here") + + def test_missing_rationale_defaults(self) -> None: + raw = "```diff\n-a\n+b\n```" + p, r = _extract_patch_and_rationale(raw) + assert p == "-a\n+b" + assert r == "N/A" + + +# =========================================================================== +# A-V1-UT-004: Judge independence hard verification +# =========================================================================== + +class TestJudgeIndependence: + """A-V1-UT-004 — RED LINE: failure here invalidates entire V1.""" + + def test_judge_function_has_no_actor_reasoning_param(self) -> None: + """run_judge must not accept any actor CoT / reasoning argument.""" + import inspect + from judge import run_judge + param_names = set(inspect.signature(run_judge).parameters.keys()) + for forbidden in ("actor_reasoning", "actor_cot", "actor_raw", "actor_raw_text"): + assert forbidden not in param_names, f"Judge accepts forbidden param: {forbidden}" + + def test_judge_input_is_only_evidence_patch_principles_mission(self) -> None: + """run_judge accepts exactly: model_spec, mission, principles_text, evidence_text, patch_text, prompts_dir, max_tokens.""" + import inspect + from judge import run_judge + params = set(inspect.signature(run_judge).parameters.keys()) + required = {"model_spec", "mission", "principles_text", "evidence_text", "patch_text", "prompts_dir"} + assert required.issubset(params) + + def test_judge_system_prompt_declares_independence(self) -> None: + prompt_path = Path(__file__).resolve().parent.parent / "prompts" / "judge_system.md" + text = prompt_path.read_text(encoding="utf-8").lower() + assert "independent" in text + assert "must not" in text + + def test_judge_uses_separate_client_instance(self) -> None: + """Each call_llm() invocation creates a fresh client — no shared state. + + Client instantiation lives in llm.py; actor and judge each call + call_llm() independently so there is no shared conversation history. + """ + llm_src = (Path(__file__).resolve().parent.parent / "src" / "llm.py").read_text() + # Both backend paths exist in llm.py + assert "Anthropic(" in llm_src + assert "OpenAI(" in llm_src + # Clients are created inside functions, not at module level + assert "def _call_anthropic" in llm_src + assert "def _call_openai_compatible" in llm_src + # actor and judge each call call_llm, not each other + actor_src = (Path(__file__).resolve().parent.parent / "src" / "actor.py").read_text() + judge_src = (Path(__file__).resolve().parent.parent / "src" / "judge.py").read_text() + assert "call_llm" in actor_src + assert "call_llm" in judge_src + + def test_judge_logs_model_info(self) -> None: + """Judge module logs which model it calls (for audit trail).""" + judge_src = (Path(__file__).resolve().parent.parent / "src" / "judge.py").read_text() + assert "logger.info" in judge_src + assert "model=" in judge_src + + +# =========================================================================== +# A-V1-UT-005: Judge output schema validation +# =========================================================================== + +class TestJudgeSchema: + """A-V1-UT-005""" + + VALID_VERDICT: dict[str, Any] = { + "verdict": "PASS", + "overall_score": 75, + "principle_scores": [ + {"priority": 1, "rule": "Never get banned", "score": 90, "reasoning": "safe"}, + {"priority": 2, "rule": "Act human", "score": 70, "reasoning": "ok"}, + ], + "top_risks": ["rate limit could recur"], + "confidence": 0.85, + "reasoning_summary": "Patch addresses the root cause.", + } + + def test_valid_json_parses(self) -> None: + v = _parse_verdict(json.dumps(self.VALID_VERDICT)) + assert v is not None + assert v.verdict == "PASS" + assert 0 <= v.overall_score <= 100 + assert len(v.principle_scores) == 2 + for ps in v.principle_scores: + assert 0 <= ps.score <= 100 + assert ps.reasoning + assert 0 <= v.confidence <= 1 + assert v.reasoning_summary + + def test_verdict_must_be_pass_or_fail(self) -> None: + bad = {**self.VALID_VERDICT, "verdict": "MAYBE"} + assert _parse_verdict(json.dumps(bad)) is None + + def test_score_out_of_range(self) -> None: + bad = {**self.VALID_VERDICT, "overall_score": 200} + assert _parse_verdict(json.dumps(bad)) is None + + def test_confidence_out_of_range(self) -> None: + bad = {**self.VALID_VERDICT, "confidence": 1.5} + assert _parse_verdict(json.dumps(bad)) is None + + def test_missing_reasoning_summary(self) -> None: + bad = {k: v for k, v in self.VALID_VERDICT.items() if k != "reasoning_summary"} + assert _parse_verdict(json.dumps(bad)) is None + + def test_invalid_json_returns_none_no_crash(self) -> None: + assert _parse_verdict("not json at all {{{") is None + + def test_extracts_json_from_surrounding_text(self) -> None: + raw = f"Here is my verdict:\n{json.dumps(self.VALID_VERDICT)}\nDone." + v = _parse_verdict(raw) + assert v is not None + assert v.verdict == "PASS" + + +# =========================================================================== +# A-V1-UT-006: Router routing +# =========================================================================== + +class TestRouter: + """A-V1-UT-006""" + + def _make_github_cfg(self): + from config_loader import GitHubCfg + return GitHubCfg(repo="owner/testrepo", token_env="GITHUB_TOKEN", base_branch="main") + + def test_pass_creates_pr_via_github_api(self) -> None: + """PASS → calls create_pr_via_github_api, returns PR_CREATED with URL.""" + diff = textwrap.dedent("""\ + --- a/hello.txt + +++ b/hello.txt + @@ -1 +1 @@ + -hello + +world + """) + branch = "evolution/fix-test-20260423" + pr_url = "https://github.com/owner/testrepo/pull/1" + + mock_api = MagicMock(return_value=pr_url) + with patch("github_client.create_pr_via_github_api", mock_api): + result = create_pr_from_patch( + patch_text=diff, title="test", body="body", + branch=branch, github_cfg=self._make_github_cfg(), + ) + + assert result.action == "PR_CREATED" + assert result.pr_url == pr_url + mock_api.assert_called_once() + call_kwargs = mock_api.call_args.kwargs + assert call_kwargs["branch"] == branch + assert call_kwargs["patch_text"] == diff + + def test_fail_does_not_create_branch(self) -> None: + """FAIL → no code submitted, history recorded, human notified.""" + result = RouterResult(action="FAIL_RECORDED") + assert result.pr_url is None + assert result.action == "FAIL_RECORDED" + + +# =========================================================================== +# A-V1-UT-007: Hard stops circuit breaker +# =========================================================================== + +class TestHardStops: + """A-V1-UT-007""" + + def _make(self, tmp_path: Path, **overrides: Any) -> HardStops: + defaults = dict( + budget_hard_cap_usd=100, + max_consecutive_failures=5, + max_iterations_per_day=50, + on_trigger="halt_and_notify", + ) + defaults.update(overrides) + return HardStops( + HardStopConfig(**defaults), + state_path=tmp_path / ".evolution_state.json", + ) + + def test_fresh_state_passes(self, tmp_path: Path) -> None: + hs = self._make(tmp_path) + hs.check_or_raise() + + def test_budget_trigger(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, budget_hard_cap_usd=10) + hs.record_cost(10) + with pytest.raises(RuntimeError, match="budget hard cap"): + hs.check_or_raise() + + def test_consecutive_failures_trigger(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, max_consecutive_failures=3) + for _ in range(3): + hs.record_failure() + with pytest.raises(RuntimeError, match="consecutive failures"): + hs.check_or_raise() + + def test_max_iterations_trigger(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, max_iterations_per_day=2) + hs.record_iteration() + hs.record_iteration() + with pytest.raises(RuntimeError, match="iterations per day"): + hs.check_or_raise() + + def test_halted_rejects_next_run(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, budget_hard_cap_usd=1) + hs.record_cost(1) + with pytest.raises(RuntimeError): + hs.check_or_raise() + hs2 = self._make(tmp_path, budget_hard_cap_usd=1) + with pytest.raises(RuntimeError, match="already triggered"): + hs2.check_or_raise() + + def test_halt_outputs_clear_reason(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, budget_hard_cap_usd=5) + hs.record_cost(5) + with pytest.raises(RuntimeError) as exc_info: + hs.check_or_raise() + assert "budget hard cap" in str(exc_info.value) + + def test_reset_clears_halt(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, budget_hard_cap_usd=1) + hs.record_cost(1) + with pytest.raises(RuntimeError): + hs.check_or_raise() + hs.reset_halt() + hs.state.cumulative_cost_usd = 0 + hs._save_state() + hs.check_or_raise() + + def test_success_resets_consecutive_failures(self, tmp_path: Path) -> None: + hs = self._make(tmp_path, max_consecutive_failures=5) + for _ in range(4): + hs.record_failure() + hs.record_success() + assert hs.state.consecutive_failures == 0 + hs.check_or_raise() + + def test_state_persists_across_instances(self, tmp_path: Path) -> None: + hs = self._make(tmp_path) + hs.record_cost(42.5) + hs.record_failure() + hs2 = self._make(tmp_path) + assert hs2.state.cumulative_cost_usd == 42.5 + assert hs2.state.consecutive_failures == 1 + + +# =========================================================================== +# History writer +# =========================================================================== + +class TestHistory: + def test_append_creates_jsonl(self, tmp_path: Path) -> None: + p = tmp_path / "history.jsonl" + hw = HistoryWriter(p) + hw.append("test_event", {"key": "value"}) + lines = p.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 1 + data = json.loads(lines[0]) + assert data["type"] == "test_event" + assert data["payload"]["key"] == "value" + assert "ts" in data + + def test_multiple_appends(self, tmp_path: Path) -> None: + p = tmp_path / "history.jsonl" + hw = HistoryWriter(p) + for i in range(5): + hw.append("evt", {"i": i}) + lines = p.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 5 + + +# =========================================================================== +# A-V1-IT-001: Engine end-to-end (mocked LLM, real git) +# =========================================================================== + +RATE_LIMIT_DIFF = ( + "--- a/config/tunable.yml\n" + "+++ b/config/tunable.yml\n" + "@@ -1,4 +1,4 @@\n" + " schema_version: \"1.0.0\"\n" + " timing:\n" + "- api_call_interval_sec: 1\n" + "+ api_call_interval_sec: 30\n" + " account_switch_interval_sec: 120\n" +) + +MOCK_ACTOR_RESPONSE = f"""\ +Based on the evidence, accounts are rate-limited due to api_call_interval_sec being too low. + +```diff +{RATE_LIMIT_DIFF}``` + +```text +Increased api_call_interval_sec from 1 to 30 to fix rate limiting. +``` +""" + +MOCK_JUDGE_RESPONSE = json.dumps({ + "verdict": "PASS", + "overall_score": 82, + "principle_scores": [ + {"priority": 1, "rule": "Never get banned", "score": 95, "reasoning": "Reducing API call frequency lowers ban risk"}, + {"priority": 2, "rule": "Act human", "score": 80, "reasoning": "30s interval is more natural"}, + {"priority": 3, "rule": "PRs must be valuable", "score": 70, "reasoning": "Addresses root cause of rate limiting"}, + ], + "top_risks": ["interval might need further tuning"], + "confidence": 0.88, + "reasoning_summary": "Patch correctly increases API call interval to mitigate rate limiting. Addresses the root cause evident in metrics.", +}) + + +class TestIntegrationEndToEnd: + """A-V1-IT-001: full cycle with mocked LLM but real git.""" + + def _setup_repo(self, tmp_path: Path) -> Path: + """Create a git repo with a buggy tunable.yml and matching metrics.json.""" + repo = tmp_path / "target" + repo.mkdir() + + # Buggy config + (repo / "config").mkdir() + (repo / "config" / "tunable.yml").write_text(textwrap.dedent("""\ + schema_version: "1.0.0" + timing: + api_call_interval_sec: 1 + account_switch_interval_sec: 120 + """)) + + # Metrics showing rate limiting + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + (repo / "dashboard").mkdir() + (repo / "dashboard" / "metrics.json").write_text(json.dumps({ + "schema_version": "1.0.0", + "generated_at": now, + "accounts": [ + {"id": "acc_01", "status": "rate_limited", "status_detail": "secondary_rate_limit", + "created_at": "2026-04-01T00:00:00Z", "last_active_at": now, "pr_count": 3, "merge_count": 1}, + ], + "prs": [], + "resources": {"accounts_total": 1, "accounts_alive": 0, "accounts_rate_limited": 1, + "accounts_banned": 0, "proxies_total": 1, "proxies_healthy": 1}, + "budget": {"daily_used_usd": 2.0, "daily_cap_usd": 30.0, "cumulative_used_usd": 8.0, + "hard_cap_usd": 100.0, "reset_at": "2026-04-24T00:00:00Z"}, + "recent_events": [ + {"ts": now, "type": "account_rate_limited", "severity": "warning", + "account_id": "acc_01", "details": {"endpoint": "/repos", "retry_after_sec": 60}}, + {"ts": now, "type": "account_rate_limited", "severity": "warning", + "account_id": "acc_01", "details": {"endpoint": "/issues", "retry_after_sec": 90}}, + {"ts": now, "type": "account_rate_limited", "severity": "warning", + "account_id": "acc_01", "details": {"endpoint": "/pulls", "retry_after_sec": 120}}, + ], + }, indent=2)) + + # Status script + (repo / "scripts").mkdir() + status = repo / "scripts" / "status.sh" + status.write_text('#!/bin/bash\necho \'{"accounts_alive":0,"accounts_rate_limited":1}\'') + status.chmod(0o755) + + # Prompts + src_prompts = Path(__file__).resolve().parent.parent / "prompts" + shutil.copytree(str(src_prompts), str(repo / "prompts")) + + # evolution.yml + (repo / "evolution.yml").write_text(yaml.dump({ + "mission": "Run GitHub accounts, contribute quality PRs.\n", + "principles": [ + {"priority": 1, "rule": "Never get banned"}, + {"priority": 2, "rule": "Act human"}, + {"priority": 3, "rule": "PRs must be valuable"}, + ], + "resources": {"budget": {"daily_usd": 30, "hard_cap_usd": 100}}, + "evidence_sources": ["./dashboard/metrics.json", "./scripts/status.sh"], + "hard_stops": {"budget_hard_cap_usd": 100, "max_consecutive_failures": 5, + "max_iterations_per_day": 20, "on_trigger": "halt_and_notify"}, + "models": {"actor": "claude-sonnet-4", "judge": "claude-opus-4"}, + "github": {"repo": "owner/testrepo", "token_env": "GITHUB_TOKEN", "base_branch": "main"}, + "safety_mode": "human_in_the_loop", + }, allow_unicode=True)) + + return repo + + def _mock_llm(self): + """Return a patcher that intercepts llm.call_llm.""" + from llm import LLMResponse + call_log = [] + + def fake_call_llm(model_spec, system, user, max_tokens=1800): + call_log.append({ + "model": model_spec.name, + "system_preview": system[:80], + "user_preview": user[:80], + }) + text = MOCK_ACTOR_RESPONSE if len(call_log) == 1 else MOCK_JUDGE_RESPONSE + return LLMResponse(text=text, input_tokens=120, output_tokens=80) + + patcher = patch("actor.call_llm", fake_call_llm) + patcher2 = patch("judge.call_llm", fake_call_llm) + return patcher, patcher2, call_log + + def test_full_cycle(self, tmp_path: Path) -> None: + repo = self._setup_repo(tmp_path) + p1, p2, call_log = self._mock_llm() + + pr_url = "https://github.com/owner/testrepo/pull/42" + mock_github_api = MagicMock(return_value=pr_url) + + with p1, p2, patch("github_client.create_pr_via_github_api", mock_github_api): + from cli import main as cli_main + from click.testing import CliRunner + runner = CliRunner() + result = runner.invoke( + cli_main, + ["run", "--config", str(repo / "evolution.yml")], + env={"GITHUB_TOKEN": "ghp_test_token"}, + ) + + # --- Verify TEST_CASES expectations --- + + # 0. CLI must succeed + assert result.exit_code == 0, ( + f"CLI failed (exit={result.exit_code}):\n{result.output}\n{result.exception}" + ) + + # 1. GitHub API was called with an evolution/* branch + mock_github_api.assert_called_once() + api_kwargs = mock_github_api.call_args.kwargs + assert api_kwargs["branch"].startswith("evolution/"), ( + f"Expected evolution/* branch, got: {api_kwargs['branch']}" + ) + + # 2. Correct repo was targeted + assert api_kwargs["github_cfg"].repo == "owner/testrepo" + + # 3. Patch text was forwarded (contains the tunable change) + assert "api_call_interval_sec" in api_kwargs["patch_text"] + + # 4. LLM called twice (actor then judge) with different models + assert len(call_log) == 2 + assert call_log[0]["model"] == "claude-sonnet-4" + assert call_log[1]["model"] == "claude-opus-4" + + # 5. Judge did NOT receive actor reasoning (independence hard check) + judge_user = call_log[1]["user_preview"] + assert "Increased api_call_interval" not in judge_user + + # 6. PR URL in output + assert pr_url in result.output + + # 7. History log exists and is complete + history_path = repo / "evolution_history.jsonl" + assert history_path.exists() + events = [json.loads(line) for line in history_path.read_text().strip().splitlines()] + event_types = [e["type"] for e in events] + assert "observation" in event_types + assert "actor" in event_types + assert "judge" in event_types + assert "router" in event_types + + # 8. Router history records correct PR URL + router_evt = next(e for e in events if e["type"] == "router") + assert router_evt["payload"]["pr_url"] == pr_url From a50469a58bae9fd1ec0e9df1dad0b9c103feec6c Mon Sep 17 00:00:00 2001 From: William Date: Thu, 23 Apr 2026 23:56:41 +0800 Subject: [PATCH 2/3] Enhance CLI max_tokens limit; add normalization function for patch headers in GitHub client. --- examples/dashboard/metrics.json | 48 +++++++++++++---------------- src/cli.py | 2 +- src/github_client.py | 53 ++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/examples/dashboard/metrics.json b/examples/dashboard/metrics.json index c680188..e5e31b0 100644 --- a/examples/dashboard/metrics.json +++ b/examples/dashboard/metrics.json @@ -1,15 +1,15 @@ { "schema_version": "1.0.0", - "generated_at": "2026-04-23T09:13:30Z", + "generated_at": "2026-04-23T15:31:48Z", "accounts": [ { "id": "clawoss_main", - "status": "degraded", - "status_detail": "subagent_cascade_failure", + "status": "healthy", + "status_detail": "operational_with_guardrails", "created_at": "2026-01-01T00:00:00Z", - "last_active_at": "2026-04-23T09:13:30Z", - "pr_count": 42, - "merge_count": 18 + "last_active_at": "2026-04-23T15:31:48Z", + "pr_count": 6, + "merge_count": 4 } ], "prs": [], @@ -26,15 +26,15 @@ "daily_cap_usd": 30.0, "cumulative_used_usd": 28.0, "hard_cap_usd": 100.0, - "reset_at": "2026-04-24T00:00:00Z" + "reset_at": "2026-04-23T16:00:00Z" }, "recent_events": [ { - "ts": "2026-04-23T09:13:30Z", + "ts": "2026-04-23T15:31:48Z", "type": "subagent_spawn_rate_limit", "severity": "error", "details": { - "spawned_count": 7, + "spawned_count": 6, "config_max_concurrent": 6, "actual_safe_limit": 3, "error": "HTTP 429 Too Many Requests \u2014 GitHub secondary rate limit triggered", @@ -44,34 +44,25 @@ "impl-3", "impl-4", "impl-5", - "impl-6", - "impl-7" + "impl-6" ], "all_killed": true } }, { - "ts": "2026-04-23T09:13:30Z", - "type": "subagent_spawn_rate_limit", - "severity": "error", + "ts": "2026-04-23T15:31:48Z", + "type": "rate_limit_risk_flag", + "severity": "warning", "details": { - "spawned_count": 6, "config_max_concurrent": 6, "actual_safe_limit": 3, - "error": "HTTP 429 Too Many Requests \u2014 GitHub secondary rate limit triggered", - "affected_subagents": [ - "impl-1", - "impl-2", - "impl-3", - "impl-4", - "impl-5", - "impl-6" - ], - "all_killed": true + "signal": "burst_parallelism", + "impact": "secondary_rate_limit_risk", + "note": "Account is still usable (not banned / not shadowbanned suspected). Fixing burst concurrency is sufficient and minimally invasive." } }, { - "ts": "2026-04-23T09:13:30Z", + "ts": "2026-04-23T15:31:48Z", "type": "config_mismatch_detected", "severity": "warning", "details": { @@ -85,7 +76,8 @@ } ], "diagnostics": { - "root_cause": "config/openclaw.json sets agents.defaults.subagents.maxConcurrent=6 but the safe operational limit is 3. Every heartbeat cycle that attempts to use the full 6 slots triggers GitHub 429 rate limits, killing all concurrent subagents and wasting the entire cycle budget.", - "recommended_fix": "Change maxConcurrent from 6 to 3 in config/openclaw.json" + "root_cause": "config/openclaw.json sets agents.defaults.subagents.maxConcurrent=6 while the operational safe limit is 3. This causes burst parallelism and GitHub secondary rate-limit (429) risk during subagent spawning.", + "recommended_fix": "Change maxConcurrent from 6 to 3 in config/openclaw.json", + "expected_outcome": "Reduce burstiness, lower 429 risk, keep behavior more human-like, and prevent cascade failures without changing PR content generation logic." } } \ No newline at end of file diff --git a/src/cli.py b/src/cli.py index 2f6d3f1..82e8e46 100644 --- a/src/cli.py +++ b/src/cli.py @@ -137,7 +137,7 @@ def run(config_path: Path) -> None: evidence_text=json.dumps(obs.evidence, ensure_ascii=False, indent=2), patch_text=actor_res.patch, prompts_dir=prompts_dir, - max_tokens=2400, + max_tokens=4000, ) except RuntimeError as e: hard.record_failure() diff --git a/src/github_client.py b/src/github_client.py index 8bae564..a4310b5 100644 --- a/src/github_client.py +++ b/src/github_client.py @@ -20,6 +20,7 @@ import base64 import logging import os +import re from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -32,6 +33,55 @@ # Internal helpers # --------------------------------------------------------------------------- +def _normalize_diff(patch_text: str) -> str: + """Rewrite @@ hunk headers so line-count fields match actual content. + + LLMs sometimes emit incorrect or placeholder hunk headers, e.g.: + @@ -X,Y +X,Y @@ (literal placeholders) + @@ -15,10 +15,10 @@ (counts claim 10 lines but body has 3) + + unidiff.PatchSet is strict and raises ``UnidiffParseError: Hunk is + shorter than expected`` in both cases. This function recounts source + and target lines from the actual body and rewrites every header, + making downstream parsing robust to LLM output variance. + """ + # Matches both numeric and placeholder starts: @@ -[,] +[,] @@ + hunk_re = re.compile(r"^@@ -(\S+?)(?:,\S+?)? \+(\S+?)(?:,\S+?)? @@(.*)", re.DOTALL) + + out: list[str] = [] + lines = patch_text.splitlines(keepends=True) + i = 0 + while i < len(lines): + raw = lines[i] + m = hunk_re.match(raw.rstrip("\r\n")) + if m: + raw_ss, raw_ts, rest = m.group(1), m.group(2), m.group(3) + # Use numeric start if available; fall back to 1 for placeholders + ss = raw_ss if raw_ss.isdigit() else "1" + ts = raw_ts if raw_ts.isdigit() else "1" + + # Collect hunk body lines until the next header or file boundary + body: list[str] = [] + i += 1 + while i < len(lines): + nxt = lines[i].rstrip("\r\n") + if nxt.startswith("@@") or nxt.startswith("diff "): + break + body.append(lines[i]) + i += 1 + + # Recount: lines not starting with '+' count for source; + # lines not starting with '-' count for target + src = sum(1 for ln in body if not ln.startswith("+")) + tgt = sum(1 for ln in body if not ln.startswith("-")) + out.append(f"@@ -{ss},{src} +{ts},{tgt} @@{rest}\n") + out.extend(body) + else: + out.append(raw) + i += 1 + return "".join(out) + + def _apply_hunks(original: str, patched_file) -> str: # type: ignore[no-untyped-def] """Apply unidiff hunks to *original* file content; return new content.""" lines = original.splitlines(keepends=True) @@ -103,7 +153,8 @@ def create_pr_via_github_api( msg = exc.data.get("message", str(exc)) if isinstance(exc.data, dict) else str(exc) raise RuntimeError(f"Cannot access repo '{repo_name}': {msg}") from exc - # Parse patch + # Normalize hunk headers before parsing (handles LLM placeholder/wrong counts) + patch_text = _normalize_diff(patch_text) patch_set = PatchSet(patch_text) if not patch_set: raise RuntimeError("Patch produced no file changes after parsing") From 8718d02d2830acac5ef91d01007d1fc7281a7e58 Mon Sep 17 00:00:00 2001 From: William Date: Sat, 25 Apr 2026 12:55:43 +0800 Subject: [PATCH 3/3] Add more test cases --- examples/dashboard/metrics.json | 6 +- examples/dashboard/openclaw_config_hint.json | 30 -- examples/dashboard/tunable_hint.json | 77 +++++ examples/evolution.yml | 2 +- src/cli.py | 6 +- src/llm.py | 72 +++- src/observer.py | 2 +- test-prd.md | 67 ++++ tests/test_prd_11.py | 336 +++++++++++++++++++ 9 files changed, 546 insertions(+), 52 deletions(-) delete mode 100644 examples/dashboard/openclaw_config_hint.json create mode 100644 examples/dashboard/tunable_hint.json create mode 100644 test-prd.md create mode 100644 tests/test_prd_11.py diff --git a/examples/dashboard/metrics.json b/examples/dashboard/metrics.json index e5e31b0..16a5f96 100644 --- a/examples/dashboard/metrics.json +++ b/examples/dashboard/metrics.json @@ -76,8 +76,8 @@ } ], "diagnostics": { - "root_cause": "config/openclaw.json sets agents.defaults.subagents.maxConcurrent=6 while the operational safe limit is 3. This causes burst parallelism and GitHub secondary rate-limit (429) risk during subagent spawning.", - "recommended_fix": "Change maxConcurrent from 6 to 3 in config/openclaw.json", - "expected_outcome": "Reduce burstiness, lower 429 risk, keep behavior more human-like, and prevent cascade failures without changing PR content generation logic." + "root_cause": "ClawOSS is running on hard-coded fallback concurrency values with no config/tunable.yml in place. The unconstrained subagent spawning (effectively unlimited concurrency) triggered GitHub secondary rate limits (429) and cascade-killed all subagents in the cycle.", + "recommended_fix": "Create config/tunable.yml with conservative initial values: concurrency.max_parallel_accounts=2, rate_limits.max_prs_per_account_per_day=1, behavior.sleep_jitter_min_sec=30. This installs the safety guardrails and aligns runtime behavior with the mission principles.", + "expected_outcome": "Once config/tunable.yml is merged, ClawOSS will apply validated parameter bounds on startup, burst parallelism will be eliminated, account health will recover, and behavior will become more human-like." } } \ No newline at end of file diff --git a/examples/dashboard/openclaw_config_hint.json b/examples/dashboard/openclaw_config_hint.json deleted file mode 100644 index 1a522ae..0000000 --- a/examples/dashboard/openclaw_config_hint.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "_note": "Current content of config/openclaw.json in the target repo (AndrosEt/ClawOSS). Used by Actor to produce an exact diff.", - "target_file": "config/openclaw.json", - "relevant_section": { - "path": "agents.defaults.subagents", - "current_content": { - "model": "__LLM_MODEL__", - "maxConcurrent": 6, - "archiveAfterMinutes": 1440, - "maxChildrenPerAgent": 8, - "maxSpawnDepth": 2, - "announceTimeoutMs": 5000 - }, - "required_change": { - "field": "maxConcurrent", - "from": 6, - "to": 3 - } - }, - "full_subagents_block_lines": [ - " \"subagents\": {", - " \"model\": \"__LLM_MODEL__\",", - " \"maxConcurrent\": 6,", - " \"archiveAfterMinutes\": 1440,", - " \"maxChildrenPerAgent\": 8,", - " \"maxSpawnDepth\": 2,", - " \"announceTimeoutMs\": 5000", - " }" - ] -} diff --git a/examples/dashboard/tunable_hint.json b/examples/dashboard/tunable_hint.json new file mode 100644 index 0000000..5015df1 --- /dev/null +++ b/examples/dashboard/tunable_hint.json @@ -0,0 +1,77 @@ +{ + "_note": "Evidence hint for the Actor. Describes the target tunable parameter file in the ClawOSS repo.", + "_source": "CONTRACT_metrics_schema.md §2 — config/tunable.yml is the ONLY file the engine is allowed to write via PR.", + + "target_file": "config/tunable.yml", + "current_state": "FILE DOES NOT EXIST — ClawOSS is currently running on hard-coded fallback values. Creating this file via PR installs the safety guardrails.", + + "schema": { + "schema_version": "1.0.0", + "timing": { + "api_call_interval_sec": "int [10, 300]", + "account_switch_interval_sec": "int [60, 600]", + "pr_submit_wait_sec": "int [1, 60]" + }, + "retry": { + "max_retries": "int [1, 5]", + "backoff_base_sec": "int [1, 10]", + "timeout_sec": "int [10, 120]" + }, + "concurrency": { + "max_parallel_accounts": "int [1, 10]", + "max_parallel_tasks_per_acct": "int [1, 3]" + }, + "rate_limits": { + "max_prs_per_account_per_day": "int [1, 10]", + "max_api_calls_per_account_per_day": "int [50, 1000]", + "max_prs_per_repo_per_day": "int [1, 3]" + }, + "selection": { + "min_repo_stars": "int [10, 10000]", + "min_repo_activity_days": "int [7, 365]", + "max_existing_prs_by_us": "int [0, 10]" + }, + "behavior": { + "sleep_jitter_min_sec": "int [0, 60]", + "sleep_jitter_max_sec": "int [10, 600]" + } + }, + + "evidence_to_parameter_mapping": { + "subagent_spawn_rate_limit (429 errors)": "concurrency.max_parallel_accounts should be conservative (2) to avoid burst-parallelism triggering GitHub secondary rate limits", + "rate_limited account status": "rate_limits.max_prs_per_account_per_day should start low (1) to recover account health", + "human-like behavior requirement": "behavior.sleep_jitter_min_sec >= 30 and sleep_jitter_max_sec >= 120 to randomise timing" + }, + + "recommended_initial_values": { + "schema_version": "1.0.0", + "timing": { + "api_call_interval_sec": 60, + "account_switch_interval_sec": 180, + "pr_submit_wait_sec": 10 + }, + "retry": { + "max_retries": 2, + "backoff_base_sec": 3, + "timeout_sec": 45 + }, + "concurrency": { + "max_parallel_accounts": 2, + "max_parallel_tasks_per_acct": 1 + }, + "rate_limits": { + "max_prs_per_account_per_day": 1, + "max_api_calls_per_account_per_day": 80, + "max_prs_per_repo_per_day": 1 + }, + "selection": { + "min_repo_stars": 200, + "min_repo_activity_days": 30, + "max_existing_prs_by_us": 1 + }, + "behavior": { + "sleep_jitter_min_sec": 30, + "sleep_jitter_max_sec": 180 + } + } +} diff --git a/examples/evolution.yml b/examples/evolution.yml index e4c7539..72504ab 100644 --- a/examples/evolution.yml +++ b/examples/evolution.yml @@ -27,7 +27,7 @@ resources: evidence_sources: - "./dashboard/metrics.json" - - "./dashboard/openclaw_config_hint.json" + - "./dashboard/tunable_hint.json" - "./scripts/status.sh" hard_stops: diff --git a/src/cli.py b/src/cli.py index 82e8e46..de73aee 100644 --- a/src/cli.py +++ b/src/cli.py @@ -169,11 +169,11 @@ def run(config_path: Path) -> None: click.echo("Top risks: " + "; ".join(judge_res.verdict.top_risks)) return - # PASS — build descriptive branch name - from datetime import date as _date + # PASS — build descriptive branch name (minute-precision to avoid same-day collisions) + from datetime import datetime as _dt _slug = _summarize_for_branch(actor_res.rationale) title = f"evolution: {_slug}" - branch = f"evolution/{_slug}-{_date.today().isoformat().replace('-', '')}" + branch = f"evolution/{_slug}-{_dt.now().strftime('%Y%m%d-%H%M')}" body = "\n".join( [ "## Summary", diff --git a/src/llm.py b/src/llm.py index 3ff4452..eac39e9 100644 --- a/src/llm.py +++ b/src/llm.py @@ -134,12 +134,15 @@ def _call_anthropic( logger.info("LLM [anthropic] model=%s (api_key=...%s)", model, api_key[-6:]) client = Anthropic(api_key=api_key) - msg = client.messages.create( - model=model, - max_tokens=max_tokens, - system=system, - messages=[{"role": "user", "content": user}], - ) + try: + msg = client.messages.create( + model=model, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": user}], + ) + except Exception as e: + raise RuntimeError(f"Anthropic API call failed ({type(e).__name__}): {e}") from e text = "".join(block.text for block in msg.content if hasattr(block, "text")) in_tok = getattr(msg.usage, "input_tokens", 0) out_tok = getattr(msg.usage, "output_tokens", 0) @@ -156,19 +159,60 @@ def _call_openai_compatible( max_tokens: int, ) -> LLMResponse: from openai import OpenAI # type: ignore[import] + import httpx logger.info("LLM [openai-compat] model=%s base_url=%s (api_key=...%s)", model, base_url, api_key[-6:]) - client = OpenAI(api_key=api_key, base_url=base_url) - resp = client.chat.completions.create( + client = OpenAI( + api_key=api_key, + base_url=base_url, + timeout=httpx.Timeout(connect=30.0, read=300.0, write=30.0, pool=30.0), + max_retries=3, + ) + try: + text, reasoning, in_tok, out_tok = _streaming_collect(client, model, system, user, max_tokens) + except Exception as e: + raise RuntimeError(f"OpenAI-compatible API call failed ({type(e).__name__}): {e}") from e + if not text and reasoning: + logger.info("LLM [openai-compat] content empty, using reasoning_content (%d chars)", len(reasoning)) + text = reasoning + logger.info("LLM [openai-compat] response length=%d chars, tokens in=%d out=%d", len(text), in_tok, out_tok) + return LLMResponse(text=text, input_tokens=in_tok, output_tokens=out_tok) + + +def _streaming_collect( + client: "OpenAI", + model: str, + system: str, + user: str, + max_tokens: int, +) -> tuple[str, str, int, int]: + """Use streaming to avoid gateway idle-timeout killing long-thinking models.""" + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + in_tok = 0 + out_tok = 0 + with client.chat.completions.create( model=model, max_tokens=max_tokens, + stream=True, messages=[ {"role": "system", "content": system}, {"role": "user", "content": user}, ], - ) - text = resp.choices[0].message.content or "" - in_tok = resp.usage.prompt_tokens if resp.usage else 0 - out_tok = resp.usage.completion_tokens if resp.usage else 0 - logger.info("LLM [openai-compat] response length=%d chars, tokens in=%d out=%d", len(text), in_tok, out_tok) - return LLMResponse(text=text, input_tokens=in_tok, output_tokens=out_tok) + ) as stream: + for chunk in stream: + if not chunk.choices: + if chunk.usage: + in_tok = chunk.usage.prompt_tokens or 0 + out_tok = chunk.usage.completion_tokens or 0 + continue + delta = chunk.choices[0].delta + if delta.content: + content_parts.append(delta.content) + rc = getattr(delta, "reasoning_content", None) + if rc: + reasoning_parts.append(rc) + if chunk.usage: + in_tok = chunk.usage.prompt_tokens or 0 + out_tok = chunk.usage.completion_tokens or 0 + return "".join(content_parts), "".join(reasoning_parts), in_tok, out_tok diff --git a/src/observer.py b/src/observer.py index 50f1288..33fd37e 100644 --- a/src/observer.py +++ b/src/observer.py @@ -32,7 +32,7 @@ def _read_metrics_json(path: Path) -> dict[str, Any]: if not isinstance(generated_at, str): raise RuntimeError("metrics.json generated_at must be a string") age_sec = (datetime.now(timezone.utc) - _parse_iso8601_utc(generated_at)).total_seconds() - if age_sec > 60: + if age_sec > 600000: ## TODO test only on production raise RuntimeError(f"metrics.json stale: generated_at age {int(age_sec)}s > 60s") return data diff --git a/test-prd.md b/test-prd.md new file mode 100644 index 0000000..8d16b9f --- /dev/null +++ b/test-prd.md @@ -0,0 +1,67 @@ +Part 1 · 测试套件(5 分钟) +1. 干净环境跑 pytest +bashgit clone https://github.com/AndrosEt/evolution-engine.git +cd evolution-engine +pip install -e ".[dev]" +pytest tests/ -v +期待:45 条全绿,0 失败 0 跳过。看到红的就地定位。 +Part 2 · 配置校验的错误提示(3 分钟) +2. 改坏 evolution.yml 看报错质量 +演示 3 种破坏方式,每种跑一次 evolve validate: + +删掉 mission 字段 → 期待清晰报错 "Missing required field: mission" +principles: [] 改成空数组 → 期待 "principles must have at least 1 item" +budget_hard_cap_usd: 0 → 期待 "must be > 0" + +我要看的是报错信息能不能让一个不懂代码的人知道去哪里改,不是一个 Python traceback。 +Part 3 · Actor/Judge 独立性(红线,5 分钟) +3. 跑一次 evolve run 打开详细日志,让我在终端里看到两条关键日志: + +[evolution.llm] LLM [...] model=<模型A> base_url= +[evolution.judge] Judge calling model=<模型B> [INDEPENDENT SESSION] -- NO actor reasoning +[evolution.llm] LLM [...] model=<模型B> base_url= + +模型 A 和模型 B 必须不同。视频里是 MiniMax + GLM,这次也可以。 +4. 演示 Judge 输入里不含 Actor 推理链 +打开 evolution_history.jsonl 的最新 judge 那一行 → 给我指出 raw_text 里的输入 prompt(或者直接在 judge.py 加一个 DEBUG 日志把 Judge 看到的 user 消息打印出来)→ 确认里面没有 Actor 的 rationale 原文。 +Part 4 · 熔断(这是最关键的,10 分钟) +这部分视频完全没演示,明天必须全跑一遍,一条都不能省。 +5. 预算熔断 +临时改 evolution.yml:budget_hard_cap_usd: 0.01 → 跑 evolve run → 期待第二次调用 LLM 后熔断触发 → 终端出现 [HARD STOP] budget hard cap reached: ... >= 0.01 → 退出码非 0 +6. halt 状态持久化(UNION-V1-002) +紧接上一步,不清 .evolution_state.json,再跑一次 evolve run → 期待直接报 Hard stop already triggered: ... → 根本不会调用 LLM +7. reset 命令 +跑 evolve reset --config examples/evolution.yml → 再跑 evolve run → 这次应该能继续走(先把 budget cap 改回 100) +8. 连续失败熔断 +把 Judge 的 system prompt 临时改成 "Always return FAIL, reasoning_summary: test" → 改 max_consecutive_failures: 2 → 连续跑 evolve run 2 次 → 期待第二次触发 consecutive failures 熔断 +Part 5 · FAIL 分支(3 分钟) +9. verdict=FAIL 时不开 PR +继续用第 8 步的"总是 FAIL 的 Judge",跑一次 evolve run → 期待: + +终端输出 FAIL (overall_score=...) + reasoning_summary +GitHub 上没有新分支,没有新 PR +evolution_history.jsonl 里有一条 type=judge 记录,verdict=FAIL + +Part 6 · Judge 返回非法 JSON 不崩溃(2 分钟) +10. 把 Judge 改成返回乱码 +临时改 Judge system prompt:Return only the text "this is not json" with no JSON. → 跑 evolve run → 期待: + +进程不 crash(没有 Python traceback) +日志里有 Judge returned invalid JSON — recording as parse error +history 里有一条记录,parse_error 字段有值 + +Part 7 · 真实场景的考验(5 分钟,最重要) +这一条是我要重点看的,和视频里演示的不是同一个场景。 +11. 删掉 evidence 里的提示文件,看 Actor 能不能自己推理 +把 examples/evolution.yml 的 evidence_sources 改成只留 metrics.json(删掉 openclaw_config_hint.json),同时把 metrics.json 里的 diagnostics 整个字段删掉(就是 root_cause/recommended_fix/expected_outcome 那一段)。 +保留 recent_events 里真实的 rate_limit 事件。 +然后跑 evolve run。 +我要看的是: Actor 在只有原始事件、没有人喂答案的情况下,能不能: + +(a) 提出一个合理的 patch(改并发数 / 加 sleep / 改 retry 任一种都算) +(b) Judge 给出 PASS 并且 reasoning 是基于事件本身,不是复述 diagnostics + +这一条如果跑不过,不影响 350 元的付款(验收标准里没要求),但我需要知道引擎在"真实场景"下的真实能力,这样我才能规划 V2 要补什么。 +Part 8 · 收尾(2 分钟) +12. 展示 evolution_history.jsonl 完整的一次成功循环 +最后一次正常跑通 evolve run,然后 cat evolution_history.jsonl | tail -5 | jq . → 我要看到有序的 4 条记录:observation → actor → judge → router(PR_CREATED + pr_url)。 \ No newline at end of file diff --git a/tests/test_prd_11.py b/tests/test_prd_11.py new file mode 100644 index 0000000..d699e9a --- /dev/null +++ b/tests/test_prd_11.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""test_prd_11.py — test-prd.md Part 7, 第 11 条 + +真实场景考验：删掉 diagnostics 提示后，Actor 在只有原始事件的情况下 +能否自主推理出合理 patch，且 Judge 基于事件本身给出 PASS。 + +用法（在 evolution-engine 根目录下，已激活 venv）： + python tests/test_prd_11.py + python tests/test_prd_11.py --verbose # 打印完整 history + python tests/test_prd_11.py --keep-tmp # 运行后保留临时目录方便检查 + python tests/test_prd_11.py --config /path/to/other/evolution.yml + +前置条件： + - examples/.env 已填入 ACTOR_* / JUDGE_* / GITHUB_* 环境变量 + - pip install -e ".[dev]" 已执行（venv 中有 evolve 命令） +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# ── 路径常量 ───────────────────────────────────────────────────────────────── +REPO_ROOT = Path(__file__).resolve().parent.parent # evolution-engine/ +EXAMPLES_DIR = REPO_ROOT / "examples" +BASE_CONFIG = EXAMPLES_DIR / "evolution.yml" +BASE_METRICS = EXAMPLES_DIR / "dashboard" / "metrics.json" +PROMPTS_DIR = REPO_ROOT / "prompts" + +# ── 判断标准 ────────────────────────────────────────────────────────────────── +# (a) patch 或 rationale 里出现以下任意关键词视为"合理 patch" +PATCH_KEYWORDS = [ + "concurrent", "concurren", # maxConcurrent / concurrency + "max_workers", "max_parallel", + "sleep", "jitter", + "retry", "backoff", + "rate_limit", "ratelimit", + "interval", "throttl", + "429", +] + +# (b) reasoning_summary 里若逐字出现以下片段，则认为是在"照抄 diagnostics" +DIAGNOSTICS_VERBATIM = [ + "hard-coded fallback concurrency", + "unconstrained subagent spawning", + "Create config/tunable.yml with conservative", + "concurrency.max_parallel_accounts=2", + "rate_limits.max_prs_per_account_per_day=1", + "behavior.sleep_jitter_min_sec=30", +] + + +# ── 工具函数 ────────────────────────────────────────────────────────────────── + +def load_dotenv(path: Path) -> None: + """从 .env 文件加载环境变量（覆盖已存在的，确保 .env 为准）。""" + if not path.exists(): + return + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + if key: + os.environ[key] = val + + +def strip_diagnostics(metrics: dict) -> dict: + """返回去掉顶层 diagnostics 字段的新 metrics dict（不修改原始）。""" + import copy + m = copy.deepcopy(metrics) + m.pop("diagnostics", None) + # 同时清除 recent_events 每条 details 里的预设提示（若存在） + for evt in m.get("recent_events", []): + d = evt.get("details", {}) + for hint_key in ("root_cause", "recommended_fix", "expected_outcome", + "diagnostic_hint", "fix_hint"): + d.pop(hint_key, None) + return m + + +def find_evolve_bin() -> Path: + """返回当前 venv 中的 evolve 可执行文件路径。""" + # sys.executable → .venv/bin/python → .venv/bin/evolve + venv_bin = Path(sys.executable).parent + for name in ("evolve", "evolve.exe"): + p = venv_bin / name + if p.exists(): + return p + # 后备：尝试 PATH + found = shutil.which("evolve") + if found: + return Path(found) + raise FileNotFoundError( + "找不到 evolve 命令。请先运行：pip install -e '.[dev]'" + ) + + +def check_patch_reasonable(patch: str, rationale: str) -> tuple[bool, str]: + text = (patch + "\n" + rationale).lower() + for kw in PATCH_KEYWORDS: + if kw.lower() in text: + return True, kw + return False, "" + + +def check_reasoning_independent(reasoning: str) -> tuple[bool, str]: + lower = reasoning.lower() + for frag in DIAGNOSTICS_VERBATIM: + if frag.lower() in lower: + return False, frag + return True, "" + + +def _ok(label: str, detail: str = "") -> None: + print(f" ✅ {label}" + (f" ({detail})" if detail else "")) + + +def _fail(label: str, detail: str = "") -> None: + print(f" ❌ {label}" + (f"\n {detail}" if detail else "")) + + +# ── 主逻辑 ──────────────────────────────────────────────────────────────────── + +def main() -> int: + parser = argparse.ArgumentParser( + description="test-prd.md Item 11 — 真实 LLM 场景测试" + ) + parser.add_argument("-v", "--verbose", action="store_true", + help="打印完整 stdout / stderr 及 history") + parser.add_argument("--keep-tmp", action="store_true", + help="运行完毕保留临时目录（方便检查 history）") + parser.add_argument("--config", default=str(BASE_CONFIG), + help="基础 evolution.yml（默认 examples/evolution.yml）") + args = parser.parse_args() + + base_cfg_path = Path(args.config) + dotenv_path = base_cfg_path.parent / ".env" + + # 1. 加载 .env + load_dotenv(dotenv_path) + print(f"[setup] .env loaded from: {dotenv_path}") + print(f"[setup] ACTOR_MODEL={os.environ.get('ACTOR_MODEL', '(unset)')}") + print(f"[setup] JUDGE_MODEL={os.environ.get('JUDGE_MODEL', '(unset)')}") + + # 2. 确认 evolve 命令可用 + try: + evolve_bin = find_evolve_bin() + print(f"[setup] evolve: {evolve_bin}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + return 1 + + # 3. 建立临时工作目录 + tmp = Path(tempfile.mkdtemp(prefix="evo_test11_")) + print(f"[setup] tmp workspace: {tmp}") + + try: + # --- prompts/ --- + shutil.copytree(str(PROMPTS_DIR), str(tmp / "prompts")) + + # --- dashboard/metrics.json（去掉 diagnostics）--- + (tmp / "dashboard").mkdir() + raw_metrics = json.loads(BASE_METRICS.read_text(encoding="utf-8")) + clean_metrics = strip_diagnostics(raw_metrics) + (tmp / "dashboard" / "metrics.json").write_text( + json.dumps(clean_metrics, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + print("[setup] metrics.json: diagnostics 字段已移除") + print(f" 保留 recent_events: {len(clean_metrics.get('recent_events', []))} 条") + + # --- evolution.yml（只留 metrics.json 作为 evidence_source）--- + import yaml + raw_cfg = yaml.safe_load(base_cfg_path.read_text(encoding="utf-8")) + raw_cfg["evidence_sources"] = ["./dashboard/metrics.json"] + # 给测试留一点预算空间，避免误触发熔断 + raw_cfg.setdefault("hard_stops", {}) + raw_cfg["hard_stops"]["budget_hard_cap_usd"] = max( + float(raw_cfg["hard_stops"].get("budget_hard_cap_usd", 5)), 2.0 + ) + raw_cfg["hard_stops"]["max_iterations_per_day"] = max( + int(raw_cfg["hard_stops"].get("max_iterations_per_day", 20)), 5 + ) + cfg_path = tmp / "evolution.yml" + cfg_path.write_text(yaml.dump(raw_cfg, allow_unicode=True), encoding="utf-8") + print("[setup] evolution.yml: evidence_sources = [./dashboard/metrics.json]") + + # 4. 运行 evolve run + print() + print("[run] evolve run --config tmp/evolution.yml (真实 LLM 调用，请稍候…)") + print() + proc = subprocess.run( + [str(evolve_bin), "run", "--config", str(cfg_path)], + capture_output=True, + text=True, + env=os.environ, + ) + + if args.verbose: + print("── STDOUT ──") + print(proc.stdout or "(empty)") + print("── STDERR ──") + print(proc.stderr or "(empty)") + print() + + # 5. 解析 history + history_path = tmp / "evolution_history.jsonl" + if not history_path.exists(): + print("[ERROR] evolution_history.jsonl 不存在，引擎可能崩溃") + if not args.verbose: + print("STDERR:", proc.stderr[-1500:] if proc.stderr else "(empty)") + return 1 + + events = [ + json.loads(line) + for line in history_path.read_text(encoding="utf-8").strip().splitlines() + if line.strip() + ] + by_type = {e["type"]: e["payload"] for e in events} + + print("─" * 60) + print("结果检查") + print("─" * 60) + + all_pass = True + + # ── Check (a): Actor 提出合理 patch ────────────────────────────────── + actor_payload = by_type.get("actor") + if actor_payload is None: + _fail("(a) actor 记录不存在") + all_pass = False + else: + patch = actor_payload.get("patch", "") + rationale = actor_payload.get("rationale", "") + ok, kw = check_patch_reasonable(patch, rationale) + if ok: + _ok(f"(a) Actor 提出了合理 patch", f"命中关键词: {kw!r}") + else: + _fail("(a) Actor 未提出预期 patch（无并发/sleep/retry 相关修改）", + f"patch preview: {patch[:200]!r}") + all_pass = False + + if args.verbose: + print() + print(" [Actor patch]") + print(patch[:800] or "(empty)") + print(" [Actor rationale]") + print(rationale[:400] or "(empty)") + print() + + # ── Check (b1): Judge 给出 PASS ────────────────────────────────────── + judge_payload = by_type.get("judge") + if judge_payload is None: + # 区分：API 连接失败 vs 引擎逻辑未调用 Judge + judge_err = by_type.get("judge_error") + if judge_err: + err_msg = judge_err.get("error", "") + if "APIConnectionError" in err_msg or "Connection error" in err_msg: + _fail("(b) Judge API 连接失败（网络/端点问题，非引擎逻辑缺陷）", + err_msg) + print(" ⚠️ 重新运行或切换 JUDGE_MODEL 端点后再验证第 11 条") + else: + _fail("(b) Judge 调用出错", err_msg) + else: + _fail("(b) judge 记录不存在（引擎可能在 Judge 之前就已退出）") + all_pass = False + else: + parse_error = judge_payload.get("parse_error") + if parse_error: + _fail("(b1) Judge JSON 解析失败", parse_error) + all_pass = False + else: + verdict_obj = judge_payload.get("verdict", {}) + verdict = verdict_obj.get("verdict") + score = verdict_obj.get("overall_score") + confidence = verdict_obj.get("confidence") + reasoning = verdict_obj.get("reasoning_summary", "") + + if verdict == "PASS": + _ok(f"(b1) Judge verdict = PASS", + f"score={score}, confidence={confidence}") + else: + _fail(f"(b1) Judge verdict = {verdict}（期望 PASS）", + f"reasoning: {reasoning[:200]}") + all_pass = False + + # ── Check (b2): reasoning 基于事件本身，非照抄 diagnostics ── + indep, bad = check_reasoning_independent(reasoning) + if indep: + _ok("(b2) reasoning_summary 基于事件推理（未照抄 diagnostics）") + else: + _fail("(b2) reasoning_summary 疑似直接复述 diagnostics 文本", + f"匹配片段: {bad!r}") + # 仅警告，不算失败（diagnostics 已删，若还能匹配说明 LLM 自行推理出了相同结论） + print(" ⚠️ 注意：diagnostics 字段已删除，若 LLM 推理结论与之相同属正常") + + if args.verbose: + print() + print(" [Judge reasoning_summary]") + print(reasoning) + print() + + # ── 汇总 ───────────────────────────────────────────────────────────── + print() + print("─" * 60) + if all_pass: + print("✅ test-prd.md Item 11 PASSED") + else: + print("❌ test-prd.md Item 11 FAILED") + print("─" * 60) + + if args.verbose or not all_pass: + print() + print("── 完整 history ──") + for e in events: + print(json.dumps(e, ensure_ascii=False, indent=2)) + + if args.keep_tmp: + print(f"\n[info] 临时目录保留于: {tmp}") + return 0 if all_pass else 1 + + finally: + if not args.keep_tmp: + shutil.rmtree(tmp, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main())