diff --git a/CHANGELOG.md b/CHANGELOG.md index 8803ac90..00116771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia). +- Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. ## [0.9.2] - 2026-08-18 ### Added diff --git a/docs/harbor.md b/docs/harbor.md index 9a0453e2..7d8f9f12 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -108,6 +108,25 @@ uvx --from harbor==0.15.0 harbor run \ --jobs-dir ./harbor-jobs/hermes-deepseek-flash ``` +## Kernel browser runtime (control arm) + +By default each Harbor trial runs Chromium inside its own container. Pass `--browser-runtime kernel` to the adapter to run the same tasks against one Kernel cloud browser per task instead: + +```bash +uv run clawbench-harbor-adapt \ + --output-dir ./harbor-datasets/clawbench-v2-kernel \ + --browser-runtime kernel \ + --browser-runtime-options '{"stealth": true}' \ + --task-ids v2-1134-chapter-finder-redcross \ + --overwrite +``` + +During task setup, the environment creates exactly one Kernel browser and replay, starts the ClawBench runtime server against it, and exposes only the local credential-free CDP bridge (`http://127.0.0.1:7878`) to the agent — the Kernel API key is never visible to the benchmark agent. Session identity and cleanup metadata land in `/my-info/kernel_browser.json`. During verification the provider replay is finalized, `recording.mp4` is downloaded into `/data`, and the browser is deleted (idempotently, including failure paths via a setup trap). + +Generated tasks register a pinned Playwright MCP package (`@playwright/mcp@0.0.79`) pointed at the CDP bridge, so Harbor's stock Claude Code and Codex agents drive the Kernel browser with native Playwright MCP tool calls — structurally identical to ClawBench's native Claude/Codex harnesses. + +Export `KERNEL_API_KEY` (and optionally `KERNEL_BASE_URL` for non-production gateways) before `harbor run`; no extra flags are needed. + ## Making it fast A full V2 sweep is 129 containerized browser sessions, each capped by the task's `time_limit`. Serial, that is a very long night. What actually moves the needle, in order: diff --git a/src/clawbench/eval/harbor_adapter.py b/src/clawbench/eval/harbor_adapter.py index 4639bdce..a6b463b0 100644 --- a/src/clawbench/eval/harbor_adapter.py +++ b/src/clawbench/eval/harbor_adapter.py @@ -11,11 +11,24 @@ from pathlib import Path from typing import Any +from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + _parse_options, +) from clawbench.runner.run_support.task import build_instruction, validate_task_data from clawbench.utils.paths import RUNTIME_ROOT, asset_path DEFAULT_CASES_DIR = asset_path("test-cases", "v2") STEP_NAME = "run" +HARBOR_BROWSER_RUNTIMES = ("local", "kernel") +# Remote browser runtimes connect through the runtime server's local, +# credential-free CDP bridge instead of a container-local Chromium. +REMOTE_BRIDGE_CDP_URL = "http://127.0.0.1:7878" +LOCAL_CDP_URL = "http://127.0.0.1:9223" +# Pinned Playwright MCP package Harbor's stock Claude Code and Codex agents +# use to drive the ClawBench browser through the CDP bridge. +PLAYWRIGHT_MCP_PACKAGE = "@playwright/mcp" +PLAYWRIGHT_MCP_VERSION = "0.0.79" def sanitize_task_name(raw: str) -> str: @@ -96,6 +109,16 @@ def copy_environment(env_dir: Path) -> None: copytree_filtered(RUNTIME_ROOT / "shared", env_dir / "shared") copytree_filtered(RUNTIME_ROOT / "harbor", env_dir / "harbor") (env_dir / "harbor" / "Dockerfile").unlink(missing_ok=True) + # The Kernel lifecycle scripts reuse the same provider implementation as + # the native runner. + shutil.copy2( + Path(__file__).resolve().parents[1] + / "runner" + / "run_support" + / "browser_runtime" + / "providers.py", + env_dir / "harbor" / "browser_runtime_providers.py", + ) shutil.copy2( Path(__file__).resolve().parents[1] / "runner" @@ -108,18 +131,71 @@ def copy_environment(env_dir: Path) -> None: chmod_executable(script) -def harbor_instruction(task: dict[str, Any]) -> str: +def playwright_mcp_server(cdp_url: str) -> dict[str, Any]: + return { + "name": "playwright", + "transport": "stdio", + "command": "npx", + "args": [ + "-y", + f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}", + "--cdp-endpoint", + cdp_url, + ], + } + + +def mcp_servers_toml(servers: list[dict[str, Any]]) -> str: + if not servers: + return "" + blocks = [] + for server in servers: + args = ", ".join(json.dumps(arg) for arg in server["args"]) + blocks.append( + "[[environment.mcp_servers]]\n" + f"name = {json.dumps(server['name'])}\n" + f"transport = {json.dumps(server['transport'])}\n" + f"command = {json.dumps(server['command'])}\n" + f"args = [{args}]\n" + ) + return "\n" + "\n".join(blocks) + + +def harbor_instruction(task: dict[str, Any], *, browser_runtime: str = "local") -> str: instruction = build_instruction(task) - return ( - instruction + "\n\n---\n" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + runtime_section = ( "Harbor browser runtime:\n" - "- Use the existing Chromium session exposed by Chrome DevTools Protocol.\n" - "- CDP endpoint: http://127.0.0.1:9223\n" + "- Use the existing browser session exposed by Chrome DevTools Protocol.\n" + f"- CDP endpoint: {cdp_url}\n" "- CDP environment variables are also set for the agent process: " "CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL.\n" - "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" - "- Do not launch a separate browser. Complete the task through the existing browser session.\n" - "---\n" + ) + if browser_runtime == "local": + runtime_section += "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" + restrictions = ( + "Task constraints (matching the native ClawBench harness rules):\n" + f"- Time limit: {task.get('time_limit')} minutes. The harness stops the " + "run when the limit elapses, so finish and submit before then.\n" + "- Complete the task entirely in the browser; do not launch or use any " + "other browser.\n" + "- Use only Playwright MCP browser tools plus reading files under " + "./my-info/ to accomplish the task.\n" + "- Do NOT make direct HTTP/network requests for task completion via shell " + "tools, scripts, API calls, or SMTP — every task action must go through " + "the browser.\n" + "- Submit through the browser: perform the task's final submission action " + "in the browser so the request happens on the page.\n" + "- Stop after submission: once the final action is submitted, stop and do " + "not start other work.\n" + ) + return ( + instruction + + "\n\n---\n" + + runtime_section + + "- Do not launch a separate browser. Complete the task through the existing browser session.\n" + + "---\n\n" + + restrictions ) @@ -130,12 +206,37 @@ def task_toml( dataset_name: str, timeout_sec: int, task_dir_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> str: escaped_description = json.dumps(description) escaped_dataset = json.dumps(dataset_name) escaped_source = json.dumps(task_dir_name) escaped_package = json.dumps(package_name) - return f"""schema_version = "1.3" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + kernel_env = "" + mcp_servers = "" + if browser_runtime == "kernel": + runtime_options_line = ( + f"\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = {json.dumps(browser_runtime_options)}" + if browser_runtime_options + else '\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = "${CLAWBENCH_BROWSER_RUNTIME_OPTIONS:-}"' + ) + kernel_env = ( + '\nCLAWBENCH_HARBOR_BROWSER_RUNTIME = "kernel"' + '\nKERNEL_API_KEY = "${KERNEL_API_KEY}"' + '\nKERNEL_BASE_URL = "${KERNEL_BASE_URL:-}"' + + runtime_options_line + + '\nCLAWBENCH_RECORDING_MODE = "provider-download"' + ) + mcp_servers = mcp_servers_toml([playwright_mcp_server(REMOTE_BRIDGE_CDP_URL)]) + healthcheck_command = ( + "curl -sf http://127.0.0.1:7878/api/status | grep -q '" + + '\\"eval_interceptor_ready\\":true' + + f"' && curl -sf {cdp_url}/json/version >/dev/null" + ) + return ( + f"""schema_version = "1.3" source = "clawbench-v2" artifacts = ["/data"] @@ -147,20 +248,24 @@ def task_toml( [metadata] dataset = {escaped_dataset} source_task = {escaped_source} +browser_runtime = "{browser_runtime}" [environment] build_timeout_sec = 1200.0 network_mode = "public" -workdir = "/app" +# The container root doubles as the step workdir so every exec runs with a +# cwd that exists even on overlay drivers that cannot resolve image-created +# directories during `docker exec`. +workdir = "/" [environment.env] PURELY_MAIL_API_KEY = "${{PURELY_MAIL_API_KEY}}" PURELY_MAIL_DOMAIN = "${{PURELY_MAIL_DOMAIN}}" -CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" -BROWSER_CDP_URL = "http://127.0.0.1:9223" -CDP_URL = "http://127.0.0.1:9223" -CHROME_CDP_URL = "http://127.0.0.1:9223" -PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_CDP_URL = "{cdp_url}" +BROWSER_CDP_URL = "{cdp_url}" +CDP_URL = "{cdp_url}" +CHROME_CDP_URL = "{cdp_url}" +PLAYWRIGHT_CDP_URL = "{cdp_url}"{kernel_env} CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" CLAWBENCH_JUDGE_BASE_URL = "${{CLAWBENCH_JUDGE_BASE_URL:-}}" @@ -178,33 +283,59 @@ def task_toml( timeout_sec = 180.0 [steps.healthcheck] -command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\\\"eval_interceptor_ready\\\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +command = "{healthcheck_command}" interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 2.0 start_interval_sec = 1.0 retries = 30 """ + + mcp_servers + ) -def setup_script() -> str: - return """#!/bin/bash +def setup_script(browser_runtime: str = "local") -> str: + kernel_setup = "" + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n" + ) + if browser_runtime == "kernel": + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:7878/json/version >/dev/null; then\n" + ) + kernel_setup = ( + "# Create the Kernel browser and replay before the runtime server" + " starts so it can bridge the provider CDP endpoint.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py start\n" + "export CLAWBENCH_BROWSER_CDP_URL_FILE=/tmp/clawbench-run/kernel-cdp-url\n" + "\n" + "cleanup_browser() {\n" + " /app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py cleanup || true\n" + "}\n" + "trap cleanup_browser EXIT\n" + "\n" + ) + return f"""#!/bin/bash set -euo pipefail -mkdir -p /data /logs/verifier /app/extra_info -cp /app/eval-schema.json /eval-schema.json +mkdir -p /data /logs/verifier /extra_info /app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py \ - --task-json /app/task.json \ - --extra-info-dir /app/extra_info \ - --output-dir /app/my-info + --task-json /task.json \ + --extra-info-dir /extra_info \ + --output-dir /my-info + +# Harbor installs its stock agent before step setup. Wrap that executable so +# ClawBench's existing /data/.stop-requested signal ends the agent cleanly. +/app/src/harbor/wrap-harbor-agent.sh -/app/src/harbor/start-runtime.sh +{kernel_setup}/app/src/harbor/start-runtime.sh for _ in $(seq 1 60); do - if curl -sf http://127.0.0.1:7878/api/status >/dev/null \ - && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then - rm -f /app/setup.sh +{readiness} rm -f /app/setup.sh + trap - EXIT exit 0 fi sleep 1 @@ -215,15 +346,21 @@ def setup_script() -> str: """ -def test_script() -> str: - return """#!/bin/bash +def test_script(browser_runtime: str = "local") -> str: + kernel_finalize = "" + if browser_runtime == "kernel": + kernel_finalize = ( + "# Stop the provider replay, download the recording, delete the browser.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py finalize\n" + ) + return f"""#!/bin/bash set -euo pipefail curl -sf -X POST http://127.0.0.1:7878/api/stop || true curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true sleep 2 rm -f /data/.stop-requested -rm -rf /logs/verifier/data +{kernel_finalize}rm -rf /logs/verifier/data cp -a /data /logs/verifier/data /app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py @@ -265,6 +402,8 @@ def write_harbor_task( output_name: str, org: str, dataset_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> Path: dest = output_root / output_name if dest.exists(): @@ -292,15 +431,19 @@ def write_harbor_task( dataset_name=dataset_name, timeout_sec=timeout_sec, task_dir_name=task_dir.name, + browser_runtime=browser_runtime, + browser_runtime_options=browser_runtime_options, ) ) - (step_dir / "instruction.md").write_text(harbor_instruction(task)) + (step_dir / "instruction.md").write_text( + harbor_instruction(task, browser_runtime=browser_runtime) + ) (workdir / "eval-schema.json").write_text(json.dumps(task["eval_schema"], indent=2)) (workdir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) copy_extra_info(task, task_dir, workdir / "extra_info") - write_text_executable(workdir / "setup.sh", setup_script()) + write_text_executable(workdir / "setup.sh", setup_script(browser_runtime)) (tests_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) - write_text_executable(tests_dir / "test.sh", test_script()) + write_text_executable(tests_dir / "test.sh", test_script(browser_runtime)) write_text_executable(solution_dir / "solve.sh", solve_script()) copy_environment(env_dir) return dest @@ -339,6 +482,18 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Overwrite an existing output directory", ) + parser.add_argument( + "--browser-runtime", + choices=HARBOR_BROWSER_RUNTIMES, + default="local", + help="Browser runtime for the generated tasks; kernel creates one " + "Kernel browser per task during Harbor setup", + ) + parser.add_argument( + "--browser-runtime-options", + default=None, + help="JSON object with runtime options, e.g. '{\"stealth\":true}'", + ) return parser @@ -346,6 +501,13 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) + options: dict[str, Any] = {} + if args.browser_runtime_options: + try: + options = _parse_options(args.browser_runtime_options) + except BrowserRuntimeError as exc: + parser.error(str(exc)) + cases_dir = (args.cases_dir or DEFAULT_CASES_DIR).resolve() default_cases = args.cases_dir is None if not cases_dir.exists(): @@ -384,10 +546,16 @@ def main(argv: list[str] | None = None) -> int: output_name=out_name, org=args.org, dataset_name=args.dataset_name, + browser_runtime=args.browser_runtime, + browser_runtime_options=(json.dumps(options) if options else None), ) ) print(f"Wrote {len(written)} Harbor task(s) to {output_dir}") + if args.browser_runtime != "local": + print(f"Browser runtime: {args.browser_runtime}") + if options: + print(f"Runtime options: {json.dumps(options)}") return 0 diff --git a/src/clawbench/runner/run_support/browser_runtime/providers.py b/src/clawbench/runner/run_support/browser_runtime/providers.py index 999da3a7..9a852082 100644 --- a/src/clawbench/runner/run_support/browser_runtime/providers.py +++ b/src/clawbench/runner/run_support/browser_runtime/providers.py @@ -558,6 +558,16 @@ def _delete(self, session_id: str) -> str: raise return "deleted" + def session_exists(self, session_id: str) -> bool: + """Return True if the provider still reports this browser session.""" + try: + self._request("GET", f"/browsers/{session_id}") + except _KernelApiError as e: + if e.status == 404: + return False + raise + return True + def _stop_replay(self, session_id: str, replay_id: str) -> None: try: self._request( @@ -575,13 +585,9 @@ def start(self, task: dict[str, Any], time_limit_s: int) -> BrowserSession: timeout_seconds = min(259200, max(10, time_limit_s + 120)) payload = { **self.options, + "stealth": self.options.get("stealth", True), "headless": False, "timeout_seconds": timeout_seconds, - "viewport": { - "width": 1920, - "height": 1080, - "refresh_rate": 25, - }, } try: result = self._request_json("POST", "/browsers", payload) diff --git a/src/clawbench/runtime/harbor/Dockerfile b/src/clawbench/runtime/harbor/Dockerfile index cd7c1921..d804a557 100644 --- a/src/clawbench/runtime/harbor/Dockerfile +++ b/src/clawbench/runtime/harbor/Dockerfile @@ -8,7 +8,7 @@ RUN echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01disable-sandbox \ && printf '#!/bin/sh\n/usr/bin/dpkg-statoverride.real "$@" 2>/dev/null || true\n' \ > /usr/bin/dpkg-statoverride && chmod +x /usr/bin/dpkg-statoverride \ && apt-get update && apt-get install -y --no-install-recommends \ - chromium xvfb ffmpeg socat curl git x11vnc xclip \ + chromium xvfb ffmpeg socat curl git x11vnc xclip nodejs npm \ libegl1 libgbm1 \ fonts-noto-color-emoji fonts-noto-cjk \ && mv /usr/bin/dpkg-statoverride.real /usr/bin/dpkg-statoverride \ diff --git a/src/clawbench/runtime/harbor/kernel-browser.py b/src/clawbench/runtime/harbor/kernel-browser.py new file mode 100644 index 00000000..7a346e51 --- /dev/null +++ b/src/clawbench/runtime/harbor/kernel-browser.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Kernel browser lifecycle for the ClawBench Harbor control arm. + +Mirrors the native runner's browser-runtime phases (start / finalize / +cleanup) inside a Harbor task container, reusing the same provider +implementation the native runner uses. The Kernel API key and the real +provider CDP URL stay in root-only files under /tmp/clawbench-run; the +benchmark agent only ever sees the credential-free CDP bridge exposed by +the ClawBench runtime server. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +try: # Vendored next to this script inside Harbor task environments. + from browser_runtime_providers import ( # type: ignore[import-not-found] + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) +except ImportError: # Source checkout / tests. + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) + +DATA_DIR = Path(os.environ.get("CLAWBENCH_DATA_DIR", "/data")) +STATE_DIR = Path("/tmp/clawbench-run") +STATE_FILE = STATE_DIR / "kernel-browser-state.json" +CDP_URL_FILE = STATE_DIR / "kernel-cdp-url" +METADATA_FILE = ( + Path(os.environ.get("CLAWBENCH_MY_INFO_DIR", "/my-info")) / "kernel_browser.json" +) +LIFECYCLE_FILE = DATA_DIR / "kernel-browser-lifecycle.json" +TASK_FILE = Path("/task.json") +RUNTIME_SERVER_BRIDGE_URL = "http://127.0.0.1:7878" + + +def _load_provider() -> KernelRuntimeProvider: + api_key = os.environ.get("KERNEL_API_KEY", "") + if not api_key: + raise SystemExit("KERNEL_API_KEY is required for the kernel browser runtime") + options: dict[str, Any] = {} + raw_options = os.environ.get("CLAWBENCH_BROWSER_RUNTIME_OPTIONS", "") + if raw_options: + options = json.loads(raw_options) + if not isinstance(options, dict): + raise SystemExit("CLAWBENCH_BROWSER_RUNTIME_OPTIONS must be a JSON object") + return KernelRuntimeProvider( + api_key=api_key, + options=options, + api_url=os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com", + ) + + +def _load_state() -> dict[str, Any] | None: + if STATE_FILE.is_file(): + return json.loads(STATE_FILE.read_text()) + # Fall back to the agent-visible metadata so cleanup still works if the + # state file was lost but the browser was created. + if METADATA_FILE.is_file(): + metadata = json.loads(METADATA_FILE.read_text()) + if metadata.get("session_id"): + return {"metadata": metadata, "events": []} + return None + + +def _save_state(state: dict[str, Any]) -> None: + STATE_DIR.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(json.dumps(state, indent=2)) + STATE_FILE.chmod(0o600) + if state.get("cdp_url"): + CDP_URL_FILE.write_text(str(state["cdp_url"])) + CDP_URL_FILE.chmod(0o600) + + +def _public_metadata(state: dict[str, Any]) -> dict[str, Any]: + """Credential-free view of the session for the agent-visible file.""" + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return { + "provider": "kernel", + "mode": "remote", + "runtime": "kernel", + "session_id": metadata.get("session_id"), + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + "region": inner.get("region") or metadata.get("region"), + "stealth": ( + inner["stealth"] if "stealth" in inner else metadata.get("stealth") + ), + "timeout_seconds": ( + inner.get("timeout_seconds") or metadata.get("timeout_seconds") + ), + "cdp_url": redact_cdp_url(metadata["cdp_url"]) + if metadata.get("cdp_url") + else None, + "viewer_url": "[REDACTED]" if metadata.get("viewer_url") else None, + "cdp_bridge_url": RUNTIME_SERVER_BRIDGE_URL, + "recording_mode": "provider-download", + "status": state.get("status"), + "cleanup_status": state.get("cleanup_status"), + "cleanup_error": state.get("cleanup_error"), + "deletion_verified": state.get("deletion_verified"), + "events": state.get("events", []), + } + + +def _write_metadata(state: dict[str, Any]) -> None: + METADATA_FILE.parent.mkdir(parents=True, exist_ok=True) + METADATA_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def _record(state: dict[str, Any], event: str, **fields: Any) -> None: + entry = {"event": event, "ts": time.time(), **fields} + state.setdefault("events", []).append(entry) + print(json.dumps(entry), flush=True) + + +def _task_time_limit_s() -> int: + task = json.loads(TASK_FILE.read_text()) + return int(float(task["time_limit"]) * 60) + + +def cmd_start() -> int: + if STATE_FILE.is_file(): + state = json.loads(STATE_FILE.read_text()) + if state.get("status") == "created": + print("Kernel browser already created for this trial", flush=True) + _write_metadata(state) + return 0 + + provider = _load_provider() + task = json.loads(TASK_FILE.read_text()) + session = provider.start(task, _task_time_limit_s()) + state: dict[str, Any] = { + "status": "created", + "cdp_url": session.cdp_url, + "metadata": session.to_metadata(), + "events": [], + } + # to_metadata redacts the CDP URL; keep the real one for the runtime + # server only, in the 0600 state file. + state["metadata"]["cdp_url"] = session.cdp_url + _record(state, "browser_created", session_id=session.session_id) + _save_state(state) + _write_metadata(state) + print( + f"Kernel browser ready; CDP bridge at {RUNTIME_SERVER_BRIDGE_URL}", flush=True + ) + return 0 + + +def _download_recording(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + staging = STATE_DIR / "finalize-output" + shutil.rmtree(staging, ignore_errors=True) + session = _session_from_state(state) + provider.finalize(session, staging) + recording = staging / "data" / "recording.mp4" + if recording.is_file(): + dest = DATA_DIR / "recording.mp4" + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(recording), dest) + state["recording_bytes"] = dest.stat().st_size + + +def _session_from_state(state: dict[str, Any]) -> BrowserSession: + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return BrowserSession( + provider="kernel", + mode="remote", + session_id=metadata.get("session_id"), + cdp_url=metadata.get("cdp_url", ""), + metadata={ + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + }, + recording_mode="provider-download", + ) + + +def cmd_finalize() -> int: + state = _load_state() + if state is None: + print("No Kernel browser was created; nothing to finalize", flush=True) + return 0 + if state.get("status") == "deleted": + print("Kernel browser already finalized and deleted", flush=True) + return 0 + + provider = _load_provider() + session_id = state.get("metadata", {}).get("session_id") + try: + _download_recording(provider, state) + _record(state, "replay_finalized", recording_bytes=state.get("recording_bytes")) + state["status"] = "finalized" + _write_metadata(state) + except BrowserRuntimeError as e: + # The replay is lost, but deletion must still proceed. + state["cleanup_error"] = f"replay finalization failed: {e}" + _record(state, "replay_finalization_failed", error=str(e)) + + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "finalize_failed", error=str(e)) + _save_state(state) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record( + state, + "browser_deleted", + session_id=session_id, + cleanup_status=state.get("cleanup_status"), + deletion_verified=state.get("deletion_verified"), + ) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def cmd_cleanup() -> int: + """Cleanup-only path (e.g. setup failed after the browser was created).""" + state = _load_state() + if state is None or state.get("status") == "deleted": + print("No Kernel browser to clean up", flush=True) + return 0 + + provider = _load_provider() + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "cleanup_failed", error=str(e)) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record(state, "browser_deleted", cleanup_status=state.get("cleanup_status")) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def _verify_deletion(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + session_id = state.get("metadata", {}).get("session_id") + if not session_id: + state["deletion_verified"] = False + return + try: + state["deletion_verified"] = not provider.session_exists(session_id) + except BrowserRuntimeError as e: + state["deletion_verified"] = False + state["cleanup_error"] = f"deletion check failed: {e}" + + +def _write_lifecycle(state: dict[str, Any]) -> None: + LIFECYCLE_FILE.parent.mkdir(parents=True, exist_ok=True) + LIFECYCLE_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["start", "finalize", "cleanup"]) + args = parser.parse_args() + return {"start": cmd_start, "finalize": cmd_finalize, "cleanup": cmd_cleanup}[ + args.command + ]() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/clawbench/runtime/harbor/start-runtime.sh b/src/clawbench/runtime/harbor/start-runtime.sh index 2e0749d3..9232b87f 100644 --- a/src/clawbench/runtime/harbor/start-runtime.sh +++ b/src/clawbench/runtime/harbor/start-runtime.sh @@ -8,16 +8,32 @@ if [ -f /tmp/clawbench-run/runtime.pid ] && kill -0 "$(cat /tmp/clawbench-run/ru exit 0 fi -export DISPLAY="${DISPLAY:-:99}" -Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & -echo "$!" > /tmp/clawbench-run/xvfb.pid -sleep 1 +# Remote browser runtimes (e.g. Kernel) hand us a provider CDP URL through a +# file instead of launching local Chromium. The runtime server proxies that +# WebSocket behind a credential-free local bridge on port 7878. +REMOTE_MODE=false +if [ -n "${CLAWBENCH_BROWSER_CDP_URL_FILE:-}" ] && [ -f "${CLAWBENCH_BROWSER_CDP_URL_FILE}" ]; then + REMOTE_MODE=true +fi cd /app/src/runtime-server uv run --no-sync uvicorn server:app --host 0.0.0.0 --port 7878 >/tmp/clawbench-run/runtime-server.log 2>&1 & echo "$!" > /tmp/clawbench-run/runtime-server.pid sleep 1 +if [ "$REMOTE_MODE" = true ]; then + # Provider replays replace local X11 recording. + export CLAWBENCH_RECORDING_MODE="${CLAWBENCH_RECORDING_MODE:-provider-download}" + echo "CDP bridge ready at http://127.0.0.1:7878" + echo "$$" > /tmp/clawbench-run/runtime.pid + exit 0 +fi + +export DISPLAY="${DISPLAY:-:99}" +Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & +echo "$!" > /tmp/clawbench-run/xvfb.pid +sleep 1 + mkdir -p /tmp/chrome-profile/Default cat > /tmp/chrome-profile/Default/Preferences <<'PREFS' { diff --git a/src/clawbench/runtime/harbor/verify.py b/src/clawbench/runtime/harbor/verify.py index 3503f56d..59ac2f99 100644 --- a/src/clawbench/runtime/harbor/verify.py +++ b/src/clawbench/runtime/harbor/verify.py @@ -35,8 +35,16 @@ def write_reward( out = output_dir out.mkdir(parents=True, exist_ok=True) result = {"reward": reward, **payload} + metrics = { + "reward": reward, + "intercepted": float(bool(payload.get("intercepted"))), + } + judge_match = payload.get("judge_match") + if isinstance(judge_match, bool): + metrics["judge_match"] = float(judge_match) + (out / "reward.txt").write_text(str(reward)) - (out / "reward.json").write_text(json.dumps(result, indent=2, ensure_ascii=False)) + (out / "reward.json").write_text(json.dumps(metrics, indent=2)) (out / "clawbench-result.json").write_text( json.dumps(result, indent=2, ensure_ascii=False) ) diff --git a/src/clawbench/runtime/harbor/wrap-harbor-agent.sh b/src/clawbench/runtime/harbor/wrap-harbor-agent.sh new file mode 100755 index 00000000..50cb7509 --- /dev/null +++ b/src/clawbench/runtime/harbor/wrap-harbor-agent.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -euo pipefail + +wrap_agent() { + local executable=$1 + local path real + + path=$(command -v "$executable" 2>/dev/null || true) + if [[ -z "$path" && -x "$HOME/.local/bin/$executable" ]]; then + path="$HOME/.local/bin/$executable" + fi + [[ -n "$path" && -x "$path" ]] || return 0 + + real="${path}.clawbench-real" + if [[ ! -e "$real" ]]; then + mv "$path" "$real" + fi + + cat >"$path" <<'WRAPPER' +#!/bin/bash +set +e + +real="${BASH_SOURCE[0]}.clawbench-real" +stop_file=${CLAWBENCH_STOP_FILE:-/data/.stop-requested} +stop_result=${CLAWBENCH_STOP_RESULT:-/data/agent-stop.json} +rm -f "$stop_file" "$stop_result" + +"$real" "$@" <&0 & +agent_pid=$! +( + while kill -0 "$agent_pid" 2>/dev/null; do + if [[ -f "$stop_file" ]]; then + detected_at=$(date +%s.%N) + printf '{"stop_detected_at":%s,"signal":"INT"}\n' "$detected_at" >"$stop_result" + kill -INT "$agent_pid" 2>/dev/null || true + for _ in $(seq 1 20); do + kill -0 "$agent_pid" 2>/dev/null || break + sleep 0.1 + done + if kill -0 "$agent_pid" 2>/dev/null; then + kill -TERM "$agent_pid" 2>/dev/null || true + fi + exit 0 + fi + sleep 0.1 + done +) & +watcher_pid=$! + +wait "$agent_pid" +status=$? +kill "$watcher_pid" 2>/dev/null || true +wait "$watcher_pid" 2>/dev/null || true + +if [[ -f "$stop_file" ]]; then + exit 0 +fi +exit "$status" +WRAPPER + chmod 0755 "$path" +} + +if (( $# > 0 )); then + for executable in "$@"; do + wrap_agent "$executable" + done +else + wrap_agent claude + wrap_agent codex +fi diff --git a/src/clawbench/runtime/runtime-server/server.py b/src/clawbench/runtime/runtime-server/server.py index 3de37cbe..5b9c614d 100644 --- a/src/clawbench/runtime/runtime-server/server.py +++ b/src/clawbench/runtime/runtime-server/server.py @@ -493,6 +493,7 @@ def activate_session_target(session_id, reason): if not INTERCEPTION_FILE.exists(): result = { "intercepted": True, + "intercepted_at": time.time(), "request": request_obj, "schema": eval_schema, } diff --git a/test-cases/v2/v2-1134-chapter-finder-redcross/task.json b/test-cases/v2/v2-1134-chapter-finder-redcross/task.json index 57acd1b8..aaf54986 100644 --- a/test-cases/v2/v2-1134-chapter-finder-redcross/task.json +++ b/test-cases/v2/v2-1134-chapter-finder-redcross/task.json @@ -17,8 +17,11 @@ }, "instruction": "Find the nearest Red Cross chapter to zip code 90210 using the local chapter finder on redcross.org", "eval_schema": { - "url_pattern": "redcross\\.org\\/find-your-local-chapter", - "method": "GET" + "url_pattern": "redcross\\.org/api/lookup/v1/region-mappings/90210(?:\\?|$)", + "method": "GET", + "params": { + "type": "RCO" + } }, "time_limit": 30, "extra_info": [] diff --git a/tests/test_browser_runtime.py b/tests/test_browser_runtime.py index 11dccdd0..85e6568c 100644 --- a/tests/test_browser_runtime.py +++ b/tests/test_browser_runtime.py @@ -398,7 +398,7 @@ def fake_urlopen( monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) provider = KernelRuntimeProvider( api_key="kernel-secret", - options={"stealth": True, "region": "us-east"}, + options={"region": "us-east"}, replay_poll_interval_s=0, replay_poll_timeout_s=1, ) @@ -413,7 +413,6 @@ def fake_urlopen( "region": "us-east", "headless": False, "timeout_seconds": 1920, - "viewport": {"width": 1920, "height": 1080, "refresh_rate": 25}, } assert session.provider == "kernel" assert session.recording_mode == "provider-download" diff --git a/tests/test_harbor_adapter.py b/tests/test_harbor_adapter.py index af67bee5..898cd2e0 100644 --- a/tests/test_harbor_adapter.py +++ b/tests/test_harbor_adapter.py @@ -108,7 +108,7 @@ def test_write_harbor_task_emits_expected_tree_and_extra_info(tmp_path: Path) -> assert ( config["task"]["name"] == "clawbench/v2-047-daily-life-personal-care-taskrabbit" ) - assert config["environment"]["workdir"] == "/app" + assert config["environment"]["workdir"] == "/" assert config["environment"]["network_mode"] == "public" assert config["environment"]["env"]["BROWSER_CDP_URL"] == "http://127.0.0.1:9223" assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" @@ -172,7 +172,7 @@ def test_harbor_adapter_help_without_container_runtime(tmp_path: Path) -> None: assert "usage" in (result.stdout + result.stderr).lower() -def test_harbor_verifier_reward_json_contains_metadata(tmp_path: Path) -> None: +def test_harbor_verifier_separates_metrics_from_metadata(tmp_path: Path) -> None: write_reward( 0.0, { @@ -187,11 +187,31 @@ def test_harbor_verifier_reward_json_contains_metadata(tmp_path: Path) -> None: assert (tmp_path / "reward.txt").read_text() == "0.0" reward = json.loads((tmp_path / "reward.json").read_text()) detailed = json.loads((tmp_path / "clawbench-result.json").read_text()) - assert reward == detailed assert reward == { + "reward": 0.0, + "intercepted": 1.0, + "judge_match": 0.0, + } + assert detailed == { "reward": 0.0, "intercepted": True, "judge_match": False, "reason": "wrong payload", "task_id": 47, } + + +def test_harbor_verifier_omits_unknown_judge_match_metric(tmp_path: Path) -> None: + write_reward( + 0.0, + { + "intercepted": False, + "judge_match": None, + "reason": "not intercepted", + "task_id": 47, + }, + output_dir=tmp_path, + ) + + reward = json.loads((tmp_path / "reward.json").read_text()) + assert reward == {"reward": 0.0, "intercepted": 0.0} diff --git a/tests/test_harbor_kernel_control.py b/tests/test_harbor_kernel_control.py new file mode 100644 index 00000000..d4bc621b --- /dev/null +++ b/tests/test_harbor_kernel_control.py @@ -0,0 +1,497 @@ +"""Regression tests for the Harbor ClawBench Kernel control arm.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +import pytest + +from clawbench.eval.harbor_adapter import ( + PLAYWRIGHT_MCP_PACKAGE, + PLAYWRIGHT_MCP_VERSION, + main as adapt_main, + write_harbor_task, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +KERNEL_BROWSER_SCRIPT = ( + REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / "kernel-browser.py" +) +HARBOR_AGENT_WRAPPER = ( + REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / "wrap-harbor-agent.sh" +) + + +def _task() -> dict: + return { + "metadata": { + "task_id": 1134, + "description": "Find nearest Red Cross chapter", + }, + "instruction": "Find the nearest Red Cross chapter to zip code 90210.", + "eval_schema": {"url_pattern": "redcross\\.org/chapter", "method": "GET"}, + "time_limit": 30, + "extra_info": [], + } + + +@pytest.fixture() +def adapted_kernel_task(tmp_path: Path) -> Path: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + return write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="v2-1134-chapter-finder-redcross", + org="clawbench", + dataset_name="v2", + browser_runtime="kernel", + ) + + +def test_redcross_task_intercepts_zip_lookup_not_chapter_finder_page() -> None: + task_path = ( + REPO_ROOT + / "test-cases" + / "v2" + / "v2-1134-chapter-finder-redcross" + / "task.json" + ) + schema = json.loads(task_path.read_text())["eval_schema"] + + assert not re.search( + schema["url_pattern"], + "https://www.redcross.org/find-your-local-chapter.html", + ) + assert re.search( + schema["url_pattern"], + "https://www.redcross.org/api/lookup/v1/region-mappings/90210?type=RCO", + ) + assert schema["params"] == {"type": "RCO"} + + +def test_kernel_runtime_selection_writes_bridge_env_and_pinned_mcp( + adapted_kernel_task: Path, +) -> None: + config = tomllib.loads((adapted_kernel_task / "task.toml").read_text()) + + assert config["metadata"]["browser_runtime"] == "kernel" + env = config["environment"]["env"] + assert env["CLAWBENCH_HARBOR_BROWSER_RUNTIME"] == "kernel" + assert env["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_RECORDING_MODE"] == "provider-download" + assert env["KERNEL_API_KEY"] == "${KERNEL_API_KEY}" + + servers = config["environment"]["mcp_servers"] + assert len(servers) == 1 + server = servers[0] + assert server["name"] == "playwright" + assert server["transport"] == "stdio" + assert server["command"] == "npx" + assert server["args"][0] == "-y" + # Pinned package so the control arm is reproducible. + assert server["args"][1] == f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}" + assert "--cdp-endpoint" in server["args"] + assert "http://127.0.0.1:7878" in server["args"] + + healthcheck = config["steps"][0]["healthcheck"]["command"] + assert "7878/json/version" in healthcheck + + +def test_kernel_setup_and_test_scripts_wire_lifecycle( + adapted_kernel_task: Path, +) -> None: + workdir = adapted_kernel_task / "steps" / "run" / "workdir" + tests = adapted_kernel_task / "steps" / "run" / "tests" + + setup = (workdir / "setup.sh").read_text() + assert "kernel-browser.py start" in setup + assert "export CLAWBENCH_BROWSER_CDP_URL_FILE=" in setup + assert "trap cleanup_browser EXIT" in setup + assert "kernel-browser.py cleanup" in setup + assert "wrap-harbor-agent.sh" in setup + assert "127.0.0.1:7878/json/version" in setup + + test = (tests / "test.sh").read_text() + assert "kernel-browser.py finalize" in test + + env_dir = adapted_kernel_task / "environment" + assert (env_dir / "harbor" / "kernel-browser.py").is_file() + assert (env_dir / "harbor" / "browser_runtime_providers.py").is_file() + + +@pytest.mark.skipif( + sys.platform != "linux", + reason="wrap-harbor-agent.sh runs inside Harbor's Linux containers; " + "bash signal handling differs on mac and windows runners", +) +def test_stop_wrapper_interrupts_agent_and_exits_cleanly(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake_agent = bin_dir / "claude" + started = tmp_path / "started" + stop_file = tmp_path / "stop-requested" + stop_result = tmp_path / "agent-stop.json" + fake_agent.write_text( + "#!/bin/bash\n" + "trap 'exit 130' INT\n" + 'touch "$FAKE_AGENT_STARTED"\n' + "while true; do sleep 0.1; done\n" + ) + fake_agent.chmod(0o755) + env = { + "PATH": f"{bin_dir}:/usr/bin:/bin", + "HOME": str(tmp_path), + "FAKE_AGENT_STARTED": str(started), + "CLAWBENCH_STOP_FILE": str(stop_file), + "CLAWBENCH_STOP_RESULT": str(stop_result), + } + + subprocess.run([HARBOR_AGENT_WRAPPER, "claude"], env=env, check=True) + process = subprocess.Popen([fake_agent], env=env) + deadline = time.monotonic() + 2 + while not started.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert started.exists() + + stop_file.touch() + assert process.wait(timeout=3) == 0 + assert json.loads(stop_result.read_text())["signal"] == "INT" + + +def test_local_runtime_default_has_no_kernel_hooks(tmp_path: Path) -> None: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + out = write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="local", + org="clawbench", + dataset_name="v2", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + assert config["metadata"]["browser_runtime"] == "local" + assert "mcp_servers" not in config["environment"] + assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" + assert config["environment"]["env"].get("CLAWBENCH_HARBOR_BROWSER_RUNTIME") is None + + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + assert "kernel-browser" not in setup + assert (out / "environment" / "harbor" / "browser_runtime_providers.py").is_file() + + +def test_instruction_ports_native_restrictions(adapted_kernel_task: Path) -> None: + instruction = (adapted_kernel_task / "steps" / "run" / "instruction.md").read_text() + # Native ClawBench prompt text (source of truth) survives verbatim. + assert "entirely through the browser" in instruction + assert "Do NOT use command-line tools, scripts, or direct API/SMTP calls" in ( + instruction + ) + # Ported harness restrictions. + assert "Time limit: 30 minutes" in instruction + assert "Playwright MCP browser tools" in instruction + assert "Do NOT make direct HTTP/network requests" in instruction + assert "Submit through the browser" in instruction + assert "Stop after submission" in instruction + # Credential-free bridge endpoint, never a provider URL. + assert "http://127.0.0.1:7878" in instruction + assert "ws://" not in instruction + + +def test_adapter_cli_rejects_invalid_runtime_options_json(tmp_path: Path) -> None: + with pytest.raises(SystemExit) as excinfo: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + "not-json", + ] + ) + assert excinfo.value.code == 2 + + +def test_adapter_cli_bakes_runtime_options_into_task_toml( + tmp_path: Path, +) -> None: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + '{"stealth": true}', + ] + ) + task_toml = next((tmp_path / "out").glob("*/task.toml")) + config = tomllib.loads(task_toml.read_text()) + assert config["environment"]["env"]["CLAWBENCH_BROWSER_RUNTIME_OPTIONS"] == ( + '{"stealth": true}' + ) + + +# --------------------------------------------------------------------------- +# kernel-browser.py lifecycle script +# --------------------------------------------------------------------------- + + +def _load_kernel_browser_module(monkeypatch: pytest.MonkeyPatch, home: Path): + spec = importlib.util.spec_from_file_location( + "kernel_browser_under_test", KERNEL_BROWSER_SCRIPT + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "STATE_DIR", home / "clawbench-run") + monkeypatch.setattr( + module, "STATE_FILE", home / "clawbench-run" / "kernel-browser-state.json" + ) + monkeypatch.setattr( + module, "CDP_URL_FILE", home / "clawbench-run" / "kernel-cdp-url" + ) + monkeypatch.setattr( + module, "METADATA_FILE", home / "my-info" / "kernel_browser.json" + ) + monkeypatch.setattr( + module, "LIFECYCLE_FILE", home / "data" / "kernel-browser-lifecycle.json" + ) + monkeypatch.setattr(module, "TASK_FILE", home / "task.json") + monkeypatch.setattr(module, "DATA_DIR", home / "data") + sys.modules[spec.name] = module + return module + + +class FakeSession: + def __init__(self) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserSession, + ) + + self._session = BrowserSession( + provider="kernel", + mode="remote", + session_id="sess-123", + cdp_url="wss://kernel.example/browser/sess-123/cdp?token=secret-token", + viewer_url="https://kernel.example/live/sess-123", + viewer_url_sensitive=True, + metadata={ + "replay_id": "replay-9", + "region": "us-east", + "stealth": False, + }, + recording_mode="provider-download", + ) + + def __getattr__(self, name: str): + return getattr(self._session, name) + + +class FakeProvider: + name = "kernel" + + def __init__(self, fail_cleanup: bool = False) -> None: + self.fail_cleanup = fail_cleanup + self.finalize_calls = 0 + self.cleanup_calls = 0 + self.exists_calls: list[str] = [] + self.deleted = False + + def start(self, task: dict, time_limit_s: int) -> FakeSession: + assert time_limit_s == 1800 + return FakeSession() + + def finalize(self, session: FakeSession, output_dir: Path) -> None: + self.finalize_calls += 1 + recording = Path(output_dir) / "data" / "recording.mp4" + recording.parent.mkdir(parents=True, exist_ok=True) + recording.write_bytes(b"mp4-bytes") + session.metadata["recording_bytes"] = len(b"mp4-bytes") + + def cleanup(self, session: FakeSession) -> None: + self.cleanup_calls += 1 + if self.fail_cleanup: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + raise BrowserRuntimeError("delete failed") + self.deleted = True + + def session_exists(self, session_id: str) -> bool: + self.exists_calls.append(session_id) + return not self.deleted + + +@pytest.fixture() +def kernel_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + (tmp_path / "task.json").write_text(json.dumps(_task())) + for var in ( + "KERNEL_API_KEY", + "KERNEL_BASE_URL", + "CLAWBENCH_BROWSER_RUNTIME_OPTIONS", + ): + monkeypatch.delenv(var, raising=False) + return tmp_path + + +def test_start_writes_state_and_credential_free_metadata( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("KERNEL_API_KEY", "k-test-key") + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_start() == 0 + + state = json.loads(module.STATE_FILE.read_text()) + assert state["status"] == "created" + assert state["cdp_url"].startswith("wss://kernel.example") + if sys.platform != "win32": + assert oct(module.STATE_FILE.stat().st_mode & 0o777) == "0o600" + assert oct(module.CDP_URL_FILE.stat().st_mode & 0o777) == "0o600" + + metadata = json.loads(module.METADATA_FILE.read_text()) + assert metadata["session_id"] == "sess-123" + assert metadata["replay_id"] == "replay-9" + assert metadata["stealth"] is False + assert metadata["cdp_bridge_url"] == "http://127.0.0.1:7878" + blob = module.METADATA_FILE.read_text() + assert "k-test-key" not in blob + assert "secret-token" not in blob + + +def test_finalize_downloads_recording_deletes_browser_and_is_idempotent( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + + recording = kernel_env / "data" / "recording.mp4" + assert recording.read_bytes() == b"mp4-bytes" + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert provider.cleanup_calls == 1 + assert provider.exists_calls == ["sess-123"] + + # Idempotent rerun makes no further API calls. + assert module.cmd_finalize() == 0 + assert provider.cleanup_calls == 1 + + +def test_finalize_reports_failure_when_cleanup_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider(fail_cleanup=True) + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 1 + state = json.loads(module.STATE_FILE.read_text()) + assert state["cleanup_error"] == "delete failed" + + +def test_cleanup_recovers_from_agent_visible_metadata_alone( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + module.STATE_FILE.unlink() + + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 1 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + + +def test_commands_are_noops_without_a_created_browser( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_finalize() == 0 + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 0 + + +def test_kernel_session_exists_reports_404_as_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + KernelRuntimeProvider, + _KernelApiError, + ) + + provider = KernelRuntimeProvider(api_key="k", options={}) + + def fake_request(method: str, path: str, payload=None) -> bytes: + raise _KernelApiError("not found", status=404) + + monkeypatch.setattr(provider, "_request", fake_request) + assert provider.session_exists("gone") is False + + def ok_request(method: str, path: str, payload=None) -> bytes: + if method == "GET": + return b"{}" + raise AssertionError(method) + + monkeypatch.setattr(provider, "_request", ok_request) + assert provider.session_exists("alive") is True + + +def test_finalize_still_deletes_browser_when_replay_download_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + def failing_finalize(session, output_dir): # noqa: ANN001 + raise BrowserRuntimeError("replay stuck processing") + + monkeypatch.setattr(provider, "finalize", failing_finalize) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert "replay finalization failed" in lifecycle["cleanup_error"] + assert provider.cleanup_calls == 1