From 7dce39c03bb9ce6c7c66696498db730358df3321 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:11:35 +0000 Subject: [PATCH 01/22] Add Harbor MCP benchmark harness --- .dockerignore | 8 + .gitignore | 8 + benchmarks/harbor/README.md | 49 ++++++ benchmarks/harbor/bin/kernel-mcp-local | 13 ++ benchmarks/harbor/bin/mcp-telemetry-proxy.mjs | 112 ++++++++++++++ benchmarks/harbor/bin/start-kernel-mcp-server | 63 ++++++++ benchmarks/harbor/bin/verify-smoke.py | 146 ++++++++++++++++++ benchmarks/harbor/build-image.sh | 65 ++++++++ benchmarks/harbor/image/Dockerfile | 34 ++++ benchmarks/harbor/mcp/kernel.json | 7 + benchmarks/harbor/prepare-task.py | 38 +++++ benchmarks/harbor/run-smoke.sh | 84 ++++++++++ benchmarks/harbor/smoke/environment/.gitkeep | 0 .../harbor/smoke/steps/run/instruction.md | 17 ++ .../harbor/smoke/steps/run/tests/test.sh | 4 + .../harbor/smoke/steps/run/workdir/setup.sh | 12 ++ benchmarks/harbor/smoke/task.toml | 51 ++++++ 17 files changed, 711 insertions(+) create mode 100644 .dockerignore create mode 100644 benchmarks/harbor/README.md create mode 100755 benchmarks/harbor/bin/kernel-mcp-local create mode 100644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs create mode 100755 benchmarks/harbor/bin/start-kernel-mcp-server create mode 100755 benchmarks/harbor/bin/verify-smoke.py create mode 100755 benchmarks/harbor/build-image.sh create mode 100644 benchmarks/harbor/image/Dockerfile create mode 100644 benchmarks/harbor/mcp/kernel.json create mode 100755 benchmarks/harbor/prepare-task.py create mode 100755 benchmarks/harbor/run-smoke.sh create mode 100644 benchmarks/harbor/smoke/environment/.gitkeep create mode 100644 benchmarks/harbor/smoke/steps/run/instruction.md create mode 100755 benchmarks/harbor/smoke/steps/run/tests/test.sh create mode 100755 benchmarks/harbor/smoke/steps/run/workdir/setup.sh create mode 100644 benchmarks/harbor/smoke/task.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b58cdc1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.github +.next +node_modules +coverage +.env* +*.log +benchmarks/harbor/.image.env diff --git a/.gitignore b/.gitignore index 4069f2b..4e9fb8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Dependencies node_modules/ +__pycache__/ +*.py[cod] npm-debug.log* yarn-debug.log* yarn-error.log* @@ -107,5 +109,11 @@ Makefile # private key mcp-key.pem +# Harbor benchmark runtime data +benchmarks/harbor/.image.env +benchmarks/harbor/.run.env +benchmarks/harbor/jobs/ +benchmarks/harbor/image/source-sha + # TypeScript incremental build cache tsconfig.tsbuildinfo diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md new file mode 100644 index 0000000..dcb0ae8 --- /dev/null +++ b/benchmarks/harbor/README.md @@ -0,0 +1,49 @@ +# Harbor MCP benchmarks + +This directory runs stock Harbor agents against a locally built `kernel-mcp-server` in a single Hypeman sandbox. The smoke task makes two read-only calls through the configured stdio MCP server and writes standard Harbor job artifacts. + +## Requirements + +- Harbor 0.21.0 with `harbor_hypeman:HypemanEnvironment` +- `harbor-hypeman` with existing Hypeman image-reference support +- Hypeman CLI and credentials +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project +- `KERNEL_MCP_BENCHMARK_PROJECT_ID` +- `ANTHROPIC_API_KEY` for Claude Code +- `OPENAI_API_KEY` for Codex + +## Build the image + +```bash +./benchmarks/harbor/build-image.sh +``` + +The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 60-second ready-image check for that case. + +## Run the smoke task + +```bash +export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id +./benchmarks/harbor/run-smoke.sh claude-code +./benchmarks/harbor/run-smoke.sh codex +``` + +Defaults: + +| Agent | Version | Model | +| ----------- | ------: | ---------------------------- | +| Claude Code | 2.1.110 | `claude-sonnet-4-5-20250929` | +| Codex | 0.120.0 | `gpt-5.3-codex` | + +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. + +The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: + +- `steps/run/agent/trajectory.json` in ATIF format +- native agent logs and session data +- `steps/run/artifacts/logs/kernel-mcp/requests.jsonl` with MCP latency and status +- server stdout and stderr +- source SHA and Hypeman identity in `run-manifest.json` +- numeric Harbor rewards plus detailed `smoke-result.json` + +The verifier requires native trajectory calls to `get_connection_context` and `manage_browsers`; direct HTTP or custom MCP-client workarounds do not pass. diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local new file mode 100755 index 0000000..5b79a8e --- /dev/null +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +key_file=/run/kernel-mcp-benchmark/api-key +if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then + KERNEL_API_KEY=$(cat "$key_file") + export KERNEL_API_KEY +fi +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +exec npx -y mcp-remote@0.1.38 \ + http://127.0.0.1:3002/mcp \ + --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs new file mode 100644 index 0000000..ed1f029 --- /dev/null +++ b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs @@ -0,0 +1,112 @@ +import fs from "node:fs"; +import http from "node:http"; + +const listenPort = Number(process.env.KERNEL_MCP_PROXY_PORT || 3002); +const upstreamPort = Number(process.env.KERNEL_MCP_SERVER_PORT || 3003); +const logPath = + process.env.KERNEL_MCP_REQUEST_LOG || "/logs/kernel-mcp/requests.jsonl"; + +function requestMetadata(body) { + try { + const payload = JSON.parse(body); + return { + jsonrpc_method: payload.method ?? null, + tool_name: + payload.method === "tools/call" ? (payload.params?.name ?? null) : null, + request_id: payload.id ?? null, + }; + } catch { + return { jsonrpc_method: null, tool_name: null, request_id: null }; + } +} + +function responseSucceeded(statusCode, body) { + if (statusCode < 200 || statusCode >= 300) return false; + const candidates = body + .split("\n") + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trim()); + if (candidates.length === 0) candidates.push(body.trim()); + + for (const candidate of candidates) { + if (!candidate || candidate === "[DONE]") continue; + try { + const payload = JSON.parse(candidate); + if (payload.error || payload.result?.isError === true) return false; + } catch { + // Non-JSON response bodies are successful when the HTTP status succeeded. + } + } + return true; +} + +function appendLog(entry) { + fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); +} + +const server = http.createServer((clientRequest, clientResponse) => { + const startedAt = new Date(); + const requestChunks = []; + + clientRequest.on("data", (chunk) => requestChunks.push(chunk)); + clientRequest.on("end", () => { + const requestBody = Buffer.concat(requestChunks); + const metadata = requestMetadata(requestBody.toString("utf8")); + const headers = { ...clientRequest.headers }; + headers.host = `127.0.0.1:${upstreamPort}`; + headers["content-length"] = String(requestBody.length); + + const upstreamRequest = http.request( + { + host: "127.0.0.1", + port: upstreamPort, + method: clientRequest.method, + path: clientRequest.url, + headers, + }, + (upstreamResponse) => { + const responseChunks = []; + upstreamResponse.on("data", (chunk) => responseChunks.push(chunk)); + upstreamResponse.on("end", () => { + const responseBody = Buffer.concat(responseChunks); + const statusCode = upstreamResponse.statusCode ?? 502; + clientResponse.writeHead(statusCode, upstreamResponse.headers); + clientResponse.end(responseBody); + + appendLog({ + started_at: startedAt.toISOString(), + duration_ms: Date.now() - startedAt.getTime(), + http_method: clientRequest.method, + path: clientRequest.url, + http_status: statusCode, + success: responseSucceeded( + statusCode, + responseBody.toString("utf8"), + ), + ...metadata, + }); + }); + }, + ); + + upstreamRequest.on("error", (error) => { + if (!clientResponse.headersSent) clientResponse.writeHead(502); + clientResponse.end("Bad Gateway"); + appendLog({ + started_at: startedAt.toISOString(), + duration_ms: Date.now() - startedAt.getTime(), + http_method: clientRequest.method, + path: clientRequest.url, + http_status: 502, + success: false, + error: error.message, + ...metadata, + }); + }); + upstreamRequest.end(requestBody); + }); +}); + +server.listen(listenPort, "127.0.0.1", () => { + console.log(`MCP telemetry proxy listening on 127.0.0.1:${listenPort}`); +}); diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server new file mode 100755 index 0000000..fe299ae --- /dev/null +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -0,0 +1,63 @@ +#!/bin/bash +set -euo pipefail + +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +log_dir=/logs/kernel-mcp +key_dir=/run/kernel-mcp-benchmark +mkdir -p "$log_dir" /logs/artifacts "$key_dir" +chmod 0777 "$log_dir" /logs/artifacts +chmod 0700 "$key_dir" +printf '%s' "$KERNEL_API_KEY" >"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ + --logfile "$log_dir/redis.log" --dir /tmp + +cd /opt/kernel-mcp-server +nohup ./node_modules/.bin/next start -p 3003 \ + >"$log_dir/server.stdout.log" \ + 2>"$log_dir/server.stderr.log" & +echo $! >"$log_dir/server.pid" + +nohup node /usr/local/lib/mcp-telemetry-proxy.mjs \ + >"$log_dir/proxy.stdout.log" \ + 2>"$log_dir/proxy.stderr.log" & +echo $! >"$log_dir/proxy.pid" + +for _ in $(seq 1 90); do + if curl -fsS -X POST http://127.0.0.1:3002/mcp \ + -H "Authorization: Bearer ${KERNEL_API_KEY}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + --data '{"jsonrpc":"2.0","id":"benchmark-healthcheck","method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"harbor-healthcheck","version":"1.0.0"}}}' \ + >"$log_dir/initialize-response.txt"; then + break + fi + sleep 1 +done + +if [ ! -s "$log_dir/initialize-response.txt" ]; then + echo "Kernel MCP server did not become ready" >&2 + tail -100 "$log_dir/server.stderr.log" >&2 || true + exit 1 +fi + +python3 - <<'PY' +import json +import os +import platform +from datetime import datetime, timezone +from pathlib import Path + +manifest = { + "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), + "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), + "sandbox_hostname": platform.node(), + "started_at": datetime.now(timezone.utc).isoformat(), +} +Path("/logs/kernel-mcp/run-manifest.json").write_text(json.dumps(manifest, indent=2)) +PY + +printf 'ready\n' >"$log_dir/ready" diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py new file mode 100755 index 0000000..cacc600 --- /dev/null +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def read_requests(path: Path) -> list[dict[str, Any]]: + requests = [] + try: + lines = path.read_text().splitlines() + except OSError: + return requests + for line in lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + requests.append(value) + return requests + + +def successful_call(requests: list[dict[str, Any]], name: str) -> dict[str, Any] | None: + return next( + ( + request + for request in requests + if request.get("jsonrpc_method") == "tools/call" + and request.get("tool_name") == name + and request.get("success") is True + ), + None, + ) + + +def trajectory_tool_names(trajectory: dict[str, Any] | None) -> list[str]: + names = [] + for step in (trajectory or {}).get("steps") or []: + if not isinstance(step, dict): + continue + for call in step.get("tool_calls") or []: + if isinstance(call, dict) and isinstance(call.get("function_name"), str): + names.append(call["function_name"]) + return names + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + requests = read_requests(LOGS_DIR / "kernel-mcp/requests.jsonl") + report = read_json(LOGS_DIR / "artifacts/agent-report.json") + trajectory = read_json(LOGS_DIR / "agent/trajectory.json") + manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") + + context_call = successful_call(requests, "get_connection_context") + browsers_call = successful_call(requests, "manage_browsers") + expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + report_matches = bool( + report + and report.get("get_connection_context_succeeded") is True + and report.get("manage_browsers_list_succeeded") is True + and report.get("connection_scope_kind") == "project" + and report.get("project_id") == expected_project_id + ) + source_sha_matches = bool( + manifest + and manifest.get("kernel_mcp_server_sha") + == os.environ.get("KERNEL_MCP_SOURCE_SHA") + ) + trajectory_names = trajectory_tool_names(trajectory) + trajectory_has_calls = any( + name.endswith("get_connection_context") for name in trajectory_names + ) and any(name.endswith("manage_browsers") for name in trajectory_names) + hypeman_identity_present = bool( + manifest and manifest.get("hypeman_instance_name") + ) + + checks = { + "get_connection_context": context_call is not None, + "manage_browsers_list": browsers_call is not None, + "agent_report": report_matches, + "source_sha": source_sha_matches, + "hypeman_identity": hypeman_identity_present, + "trajectory": trajectory_has_calls, + "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), + "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), + } + reward = 1.0 if all(checks.values()) else 0.0 + tool_calls = [ + { + "name": call["tool_name"], + "success": call["success"], + "duration_ms": call["duration_ms"], + "http_status": call["http_status"], + } + for call in (context_call, browsers_call) + if call is not None + ] + result = { + "reward": reward, + "checks": checks, + "tool_calls": tool_calls, + "agent_report": report, + "trajectory": { + "present": trajectory is not None, + "schema_version": (trajectory or {}).get("schema_version"), + "agent": (trajectory or {}).get("agent"), + "tool_names": trajectory_names, + }, + "run_manifest": manifest, + } + + (VERIFIER_DIR / "reward.txt").write_text(str(reward)) + (VERIFIER_DIR / "reward.json").write_text( + json.dumps( + { + "reward": reward, + "get_connection_context": float(context_call is not None), + "manage_browsers_list": float(browsers_call is not None), + "agent_report": float(report_matches), + "source_sha": float(source_sha_matches), + "hypeman_identity": float(hypeman_identity_present), + "trajectory": float(trajectory_has_calls), + }, + indent=2, + ) + ) + (VERIFIER_DIR / "smoke-result.json").write_text(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh new file mode 100755 index 0000000..30798b8 --- /dev/null +++ b/benchmarks/harbor/build-image.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" + +source_sha=$(git rev-parse HEAD) +source_sha_file=benchmarks/harbor/image/source-sha +build_log=$(mktemp) +trap 'rm -f "$source_sha_file" "$build_log"' EXIT + +printf '%s\n' "$source_sha" >"$source_sha_file" + +set +e +hypeman build \ + --file benchmarks/harbor/image/Dockerfile \ + --cpus 4 \ + --memory 8GB \ + --timeout 30m \ + . 2>&1 | tee "$build_log" +build_status=${PIPESTATUS[0]} +set -e + +build_id=$(sed -n 's/^Build ID: //p' "$build_log" | tail -1) +if [[ -z "$build_id" ]]; then + echo "Hypeman did not return a build ID" >&2 + exit 1 +fi + +image_ref="builds/$build_id" +if ((build_status != 0)); then + echo "Build record failed; checking for a delayed ready image for up to 60 seconds" >&2 + image_ready=false + for _ in $(seq 1 12); do + if hypeman --format json image list | python3 -c ' +import json +import sys + +image_ref = sys.argv[1] +expected = {image_ref, f"docker.io/{image_ref}:latest"} +images = json.load(sys.stdin) +raise SystemExit( + 0 + if any(image.get("name") in expected and image.get("status") == "ready" for image in images) + else 1 +) +' "$image_ref" + then + image_ready=true + break + fi + sleep 5 + done + if [[ "$image_ready" != true ]]; then + exit "$build_status" + fi +fi + +cat >benchmarks/harbor/.image.env < int: + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + args = parser.parse_args() + + source = Path(__file__).parent / "smoke" + output = args.output.resolve() + image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] + source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] + + if output.exists(): + shutil.rmtree(output) + shutil.copytree(source, output) + + config_path = output / "task.toml" + config = config_path.read_text() + config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) + config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) + config_path.write_text(config) + + wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" + runtime_wrapper = output / "steps" / "run" / "workdir" / "kernel-mcp-local" + shutil.copy2(wrapper, runtime_wrapper) + runtime_wrapper.chmod(0o755) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh new file mode 100755 index 0000000..bfafd5a --- /dev/null +++ b/benchmarks/harbor/run-smoke.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -euo pipefail + +usage() { + echo "usage: $0 [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} + +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" + +case "$agent" in + claude-code) + : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-4-5-20250929} + version=2.1.110 + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.3-codex} + version=0.120.0 + ;; +esac + +if [[ -n "${HARBOR_BIN:-}" ]]; then + harbor_bin=$HARBOR_BIN +elif command -v harbor >/dev/null 2>&1; then + harbor_bin=$(command -v harbor) +elif [[ -x "$repo_root/../harbor-hypeman/.venv/bin/harbor" ]]; then + harbor_bin="$repo_root/../harbor-hypeman/.venv/bin/harbor" +else + echo "Harbor CLI not found; set HARBOR_BIN" >&2 + exit 1 +fi + +job_name=${2:-${agent}-smoke-$(date -u +%Y%m%dT%H%M%SZ)} +jobs_dir=${3:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-harbor-jobs}} +runtime_task=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_task"; rm -f "$runtime_env"' EXIT + +export KERNEL_MCP_BENCHMARK_IMAGE KERNEL_MCP_SOURCE_SHA +python3 "$benchmark_dir/prepare-task.py" "$runtime_task" + +cat >"$runtime_env" <"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +/usr/local/bin/start-kernel-mcp-server +printf 'ready\n' >/logs/kernel-mcp/ready +rm -f /app/setup.sh diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml new file mode 100644 index 0000000..d3f95cf --- /dev/null +++ b/benchmarks/harbor/smoke/task.toml @@ -0,0 +1,51 @@ +schema_version = "1.4" +source = "kernel-mcp-benchmarks" +artifacts = ["/logs/kernel-mcp"] +multi_step_reward_strategy = "final" + +[task] +name = "kernel-mcp/local-connection-smoke" +description = "Verify an agent can call a locally running Kernel MCP server" +keywords = ["kernel", "mcp", "harbor", "smoke"] + +[metadata] +benchmark = "kernel-mcp-local-connection" +kernel_mcp_server_sha = "${KERNEL_MCP_SOURCE_SHA}" + +[environment] +docker_image = "${KERNEL_MCP_BENCHMARK_IMAGE}" +network_mode = "public" +workdir = "/app" +build_timeout_sec = 1200.0 +cpus = 2 +memory_mb = 4096 +storage_mb = 8192 + +[environment.env] +KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" +KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" +KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" +KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" +API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" +REDIS_URL = "redis://127.0.0.1:6379" +CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" +NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY = "pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ" +MANAGED_AUTH_APP_ORIGIN = "http://127.0.0.1:3002" +NEXT_TELEMETRY_DISABLED = "1" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 300.0 + +[steps.verifier] +timeout_sec = 60.0 + +[steps.healthcheck] +command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp && curl -sS -o /dev/null http://127.0.0.1:3003/mcp" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 1.0 +start_interval_sec = 1.0 +retries = 10 From 388d2dabf99631231302d819876fc67ac1dc2971 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:07:52 +0000 Subject: [PATCH 02/22] Update benchmark model defaults --- benchmarks/harbor/README.md | 8 ++++---- benchmarks/harbor/run-smoke.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index dcb0ae8..2462e92 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -30,10 +30,10 @@ export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id Defaults: -| Agent | Version | Model | -| ----------- | ------: | ---------------------------- | -| Claude Code | 2.1.110 | `claude-sonnet-4-5-20250929` | -| Codex | 0.120.0 | `gpt-5.3-codex` | +| Agent | Version | Model | +| ----------- | ------: | ----------------- | +| Claude Code | 2.1.110 | `claude-sonnet-5` | +| Codex | 0.120.0 | `gpt-5.6-terra` | Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index bfafd5a..47b420d 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -27,12 +27,12 @@ set +a case "$agent" in claude-code) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" - model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-4-5-20250929} + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} version=2.1.110 ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.3-codex} + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} version=0.120.0 ;; esac From 41b53f803222d01361d79598261a12d499a9e683 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:35:07 +0000 Subject: [PATCH 03/22] Fix Claude Sonnet Harbor smoke defaults --- benchmarks/harbor/README.md | 8 ++++---- benchmarks/harbor/build-image.sh | 16 ++++++++-------- benchmarks/harbor/prepare-task.py | 2 ++ benchmarks/harbor/run-smoke.sh | 17 +++++++++++++++-- benchmarks/harbor/smoke/task.toml | 1 + 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 2462e92..6a06329 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -9,7 +9,7 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv - Hypeman CLI and credentials - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` -- `ANTHROPIC_API_KEY` for Claude Code +- `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex ## Build the image @@ -18,7 +18,7 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv ./benchmarks/harbor/build-image.sh ``` -The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 60-second ready-image check for that case. +The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. ## Run the smoke task @@ -32,10 +32,10 @@ Defaults: | Agent | Version | Model | | ----------- | ------: | ----------------- | -| Claude Code | 2.1.110 | `claude-sonnet-5` | +| Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-terra` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`, or test a specific Claude Code release with `CLAUDE_BENCHMARK_VERSION`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh index 30798b8..d8319af 100755 --- a/benchmarks/harbor/build-image.sh +++ b/benchmarks/harbor/build-image.sh @@ -15,29 +15,29 @@ set +e hypeman build \ --file benchmarks/harbor/image/Dockerfile \ --cpus 4 \ - --memory 8GB \ - --timeout 30m \ + --memory 8192 \ + --timeout 1800 \ . 2>&1 | tee "$build_log" build_status=${PIPESTATUS[0]} set -e -build_id=$(sed -n 's/^Build ID: //p' "$build_log" | tail -1) +build_id=$(sed -n -E 's/^Build (ID|started): //p' "$build_log" | tail -1) if [[ -z "$build_id" ]]; then echo "Hypeman did not return a build ID" >&2 exit 1 fi -image_ref="builds/$build_id" +image_ref="docker.io/builds/$build_id:latest" if ((build_status != 0)); then - echo "Build record failed; checking for a delayed ready image for up to 60 seconds" >&2 + echo "Build record failed; checking for a delayed ready image for up to 5 minutes" >&2 image_ready=false - for _ in $(seq 1 12); do + for _ in $(seq 1 30); do if hypeman --format json image list | python3 -c ' import json import sys image_ref = sys.argv[1] -expected = {image_ref, f"docker.io/{image_ref}:latest"} +expected = {image_ref, image_ref.removeprefix("docker.io/")} images = json.load(sys.stdin) raise SystemExit( 0 @@ -49,7 +49,7 @@ raise SystemExit( image_ready=true break fi - sleep 5 + sleep 10 done if [[ "$image_ready" != true ]]; then exit "$build_status" diff --git a/benchmarks/harbor/prepare-task.py b/benchmarks/harbor/prepare-task.py index 0b031a1..6ad8107 100755 --- a/benchmarks/harbor/prepare-task.py +++ b/benchmarks/harbor/prepare-task.py @@ -16,6 +16,7 @@ def main() -> int: output = args.output.resolve() image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] + project_id = os.environ["KERNEL_MCP_BENCHMARK_PROJECT_ID"] if output.exists(): shutil.rmtree(output) @@ -25,6 +26,7 @@ def main() -> int: config = config_path.read_text() config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) + config = config.replace("${KERNEL_MCP_BENCHMARK_PROJECT_ID}", project_id) config_path.write_text(config) wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 47b420d..d4c447c 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -26,9 +26,17 @@ set +a case "$agent" in claude-code) - : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} - version=2.1.110 + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" @@ -63,6 +71,11 @@ KERNEL_MCP_SOURCE_SHA=$KERNEL_MCP_SOURCE_SHA KERNEL_MCP_BENCHMARK_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY KERNEL_MCP_BENCHMARK_PROJECT_ID=$KERNEL_MCP_BENCHMARK_PROJECT_ID KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-https://api.onkernel.com} +KERNEL_PROJECT=${KERNEL_PROJECT:-} +ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} +ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} +CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} EOF chmod 0600 "$runtime_env" diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml index d3f95cf..d2b12cb 100644 --- a/benchmarks/harbor/smoke/task.toml +++ b/benchmarks/harbor/smoke/task.toml @@ -26,6 +26,7 @@ KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" +KERNEL_PROJECT = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" REDIS_URL = "redis://127.0.0.1:6379" CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" From aaa4ac00d9dd94be0ad5bc4fd2ef7f2e05168a65 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:44:01 +0000 Subject: [PATCH 04/22] Verify Harbor MCP calls from ATIF --- benchmarks/harbor/README.md | 3 +- .../trajectory-error-observation.json | 37 +++ .../trajectory-missing-observation.json | 33 +++ .../bin/fixtures/trajectory-positive.json | 37 +++ benchmarks/harbor/bin/mcp-telemetry-proxy.mjs | 112 --------- benchmarks/harbor/bin/start-kernel-mcp-server | 7 +- benchmarks/harbor/bin/test_verify_smoke.py | 49 ++++ benchmarks/harbor/bin/verify-smoke.py | 214 ++++++++++++------ benchmarks/harbor/image/Dockerfile | 1 - benchmarks/harbor/smoke/task.toml | 2 +- 10 files changed, 300 insertions(+), 195 deletions(-) create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-error-observation.json create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-positive.json delete mode 100644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs create mode 100644 benchmarks/harbor/bin/test_verify_smoke.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 6a06329..bc08cb8 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -41,9 +41,8 @@ The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful - `steps/run/agent/trajectory.json` in ATIF format - native agent logs and session data -- `steps/run/artifacts/logs/kernel-mcp/requests.jsonl` with MCP latency and status - server stdout and stderr - source SHA and Hypeman identity in `run-manifest.json` - numeric Harbor rewards plus detailed `smoke-result.json` -The verifier requires native trajectory calls to `get_connection_context` and `manage_browsers`; direct HTTP or custom MCP-client workarounds do not pass. +The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. diff --git a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json new file mode 100644 index 0000000..99f41ef --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" + }, + { + "source_call_id": "browsers-1", + "content": "{\"isError\":true,\"content\":[{\"type\":\"text\",\"text\":\"request failed\"}]}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json new file mode 100644 index 0000000..c506f2e --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json @@ -0,0 +1,33 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-positive.json b/benchmarks/harbor/bin/fixtures/trajectory-positive.json new file mode 100644 index 0000000..a00aa56 --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-positive.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"type\":\"text\",\"text\":\"{\\\"connection_scope\\\":{\\\"kind\\\":\\\"project\\\",\\\"project_id\\\":\\\"project-123\\\"}}\"}" + }, + { + "source_call_id": "browsers-1", + "content": "{\"type\":\"text\",\"text\":\"{\\\"items\\\":[],\\\"has_more\\\":false}\"}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs deleted file mode 100644 index ed1f029..0000000 --- a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import fs from "node:fs"; -import http from "node:http"; - -const listenPort = Number(process.env.KERNEL_MCP_PROXY_PORT || 3002); -const upstreamPort = Number(process.env.KERNEL_MCP_SERVER_PORT || 3003); -const logPath = - process.env.KERNEL_MCP_REQUEST_LOG || "/logs/kernel-mcp/requests.jsonl"; - -function requestMetadata(body) { - try { - const payload = JSON.parse(body); - return { - jsonrpc_method: payload.method ?? null, - tool_name: - payload.method === "tools/call" ? (payload.params?.name ?? null) : null, - request_id: payload.id ?? null, - }; - } catch { - return { jsonrpc_method: null, tool_name: null, request_id: null }; - } -} - -function responseSucceeded(statusCode, body) { - if (statusCode < 200 || statusCode >= 300) return false; - const candidates = body - .split("\n") - .filter((line) => line.startsWith("data:")) - .map((line) => line.slice(5).trim()); - if (candidates.length === 0) candidates.push(body.trim()); - - for (const candidate of candidates) { - if (!candidate || candidate === "[DONE]") continue; - try { - const payload = JSON.parse(candidate); - if (payload.error || payload.result?.isError === true) return false; - } catch { - // Non-JSON response bodies are successful when the HTTP status succeeded. - } - } - return true; -} - -function appendLog(entry) { - fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); -} - -const server = http.createServer((clientRequest, clientResponse) => { - const startedAt = new Date(); - const requestChunks = []; - - clientRequest.on("data", (chunk) => requestChunks.push(chunk)); - clientRequest.on("end", () => { - const requestBody = Buffer.concat(requestChunks); - const metadata = requestMetadata(requestBody.toString("utf8")); - const headers = { ...clientRequest.headers }; - headers.host = `127.0.0.1:${upstreamPort}`; - headers["content-length"] = String(requestBody.length); - - const upstreamRequest = http.request( - { - host: "127.0.0.1", - port: upstreamPort, - method: clientRequest.method, - path: clientRequest.url, - headers, - }, - (upstreamResponse) => { - const responseChunks = []; - upstreamResponse.on("data", (chunk) => responseChunks.push(chunk)); - upstreamResponse.on("end", () => { - const responseBody = Buffer.concat(responseChunks); - const statusCode = upstreamResponse.statusCode ?? 502; - clientResponse.writeHead(statusCode, upstreamResponse.headers); - clientResponse.end(responseBody); - - appendLog({ - started_at: startedAt.toISOString(), - duration_ms: Date.now() - startedAt.getTime(), - http_method: clientRequest.method, - path: clientRequest.url, - http_status: statusCode, - success: responseSucceeded( - statusCode, - responseBody.toString("utf8"), - ), - ...metadata, - }); - }); - }, - ); - - upstreamRequest.on("error", (error) => { - if (!clientResponse.headersSent) clientResponse.writeHead(502); - clientResponse.end("Bad Gateway"); - appendLog({ - started_at: startedAt.toISOString(), - duration_ms: Date.now() - startedAt.getTime(), - http_method: clientRequest.method, - path: clientRequest.url, - http_status: 502, - success: false, - error: error.message, - ...metadata, - }); - }); - upstreamRequest.end(requestBody); - }); -}); - -server.listen(listenPort, "127.0.0.1", () => { - console.log(`MCP telemetry proxy listening on 127.0.0.1:${listenPort}`); -}); diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index fe299ae..08e7601 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -15,16 +15,11 @@ redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ --logfile "$log_dir/redis.log" --dir /tmp cd /opt/kernel-mcp-server -nohup ./node_modules/.bin/next start -p 3003 \ +nohup ./node_modules/.bin/next start -p 3002 \ >"$log_dir/server.stdout.log" \ 2>"$log_dir/server.stderr.log" & echo $! >"$log_dir/server.pid" -nohup node /usr/local/lib/mcp-telemetry-proxy.mjs \ - >"$log_dir/proxy.stdout.log" \ - 2>"$log_dir/proxy.stderr.log" & -echo $! >"$log_dir/proxy.pid" - for _ in $(seq 1 90); do if curl -fsS -X POST http://127.0.0.1:3002/mcp \ -H "Authorization: Bearer ${KERNEL_API_KEY}" \ diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py new file mode 100644 index 0000000..18d53be --- /dev/null +++ b/benchmarks/harbor/bin/test_verify_smoke.py @@ -0,0 +1,49 @@ +import importlib.util +import json +from pathlib import Path +import unittest + + +MODULE_PATH = Path(__file__).with_name("verify-smoke.py") +SPEC = importlib.util.spec_from_file_location("verify_smoke", MODULE_PATH) +assert SPEC and SPEC.loader +verify_smoke = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(verify_smoke) + +FIXTURES = Path(__file__).with_name("fixtures") + + +def load_fixture(name: str) -> dict: + return json.loads((FIXTURES / name).read_text()) + + +class VerifySmokeTest(unittest.TestCase): + def test_accepts_native_calls_with_paired_observations(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-positive.json"), "project-123" + ) + + self.assertTrue(proof["native_calls_present"]) + self.assertTrue(proof["observations_valid"]) + self.assertTrue(proof["context_scope_valid"]) + self.assertTrue(proof["manage_browsers_arguments_valid"]) + + def test_rejects_missing_observation(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-missing-observation.json"), "project-123" + ) + + self.assertFalse(proof["observations_valid"]) + self.assertEqual(proof["missing_observations"], ["browsers-1"]) + + def test_rejects_error_observation(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-error-observation.json"), "project-123" + ) + + self.assertFalse(proof["observations_valid"]) + self.assertEqual(proof["error_observations"], ["browsers-1"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py index cacc600..5b0de04 100755 --- a/benchmarks/harbor/bin/verify-smoke.py +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -8,6 +8,10 @@ LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) VERIFIER_DIR = LOGS_DIR / "verifier" +REQUIRED_TOOLS = { + "mcp__kernel__get_connection_context", + "mcp__kernel__manage_browsers", +} def read_json(path: Path) -> dict[str, Any] | None: @@ -18,107 +22,179 @@ def read_json(path: Path) -> dict[str, Any] | None: return value if isinstance(value, dict) else None -def read_requests(path: Path) -> list[dict[str, Any]]: - requests = [] - try: - lines = path.read_text().splitlines() - except OSError: - return requests - for line in lines: - try: - value = json.loads(line) - except json.JSONDecodeError: +def _decode_tool_result_content(content: Any) -> Any: + value = content + for _ in range(4): + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return value + continue + if isinstance(value, dict) and value.get("type") == "text": + value = value.get("text") + continue + return value + return value + + +def _contains_error(value: Any) -> bool: + if isinstance(value, dict): + if value.get("is_error") is True or value.get("isError") is True: + return True + if "error" in value and value["error"] not in (None, False, ""): + return True + return any(_contains_error(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error(item) for item in value) + if isinstance(value, str): + text = value.lstrip().lower() + return text.startswith(("[error]", "error:", "error in ")) + return False + + +def _observation_result_map(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + results: dict[str, list[dict[str, Any]]] = {} + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + observation = step.get("observation") + if not isinstance(observation, dict): continue - if isinstance(value, dict): - requests.append(value) - return requests - - -def successful_call(requests: list[dict[str, Any]], name: str) -> dict[str, Any] | None: - return next( - ( - request - for request in requests - if request.get("jsonrpc_method") == "tools/call" - and request.get("tool_name") == name - and request.get("success") is True - ), - None, + for result in observation.get("results") or []: + if not isinstance(result, dict): + continue + source_call_id = result.get("source_call_id") + if isinstance(source_call_id, str): + results.setdefault(source_call_id, []).append(result) + return results + + +def _native_tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls = [] + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + for call in step.get("tool_calls") or []: + if not isinstance(call, dict): + continue + if call.get("function_name") in REQUIRED_TOOLS: + calls.append(call) + return calls + + +def _manage_browsers_arguments_valid(call: dict[str, Any]) -> bool: + arguments = call.get("arguments") + return ( + isinstance(arguments, dict) + and arguments.get("action") == "list" + and arguments.get("status") == "active" + and arguments.get("limit") == 1 ) -def trajectory_tool_names(trajectory: dict[str, Any] | None) -> list[str]: - names = [] - for step in (trajectory or {}).get("steps") or []: - if not isinstance(step, dict): +def validate_trajectory( + trajectory: dict[str, Any] | None, expected_project_id: str +) -> dict[str, Any]: + calls = _native_tool_calls(trajectory or {}) + calls_by_name = { + name: [call for call in calls if call.get("function_name") == name] + for name in REQUIRED_TOOLS + } + result_map = _observation_result_map(trajectory or {}) + missing_observations = [] + duplicate_observations = [] + error_observations = [] + context_scope_valid = True + browser_arguments_valid = True + + for call in calls: + call_id = call.get("tool_call_id") + results = result_map.get(call_id, []) if isinstance(call_id, str) else [] + if len(results) == 0: + missing_observations.append(call_id) continue - for call in step.get("tool_calls") or []: - if isinstance(call, dict) and isinstance(call.get("function_name"), str): - names.append(call["function_name"]) - return names + if len(results) != 1: + duplicate_observations.append(call_id) + continue + result = results[0] + decoded = _decode_tool_result_content(result.get("content")) + if _contains_error(decoded) or _contains_error(result.get("extra")): + error_observations.append(call_id) + continue + if call.get("function_name") == "mcp__kernel__get_connection_context": + scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None + context_scope_valid = context_scope_valid and ( + isinstance(scope, dict) + and scope.get("kind") == "project" + and scope.get("project_id") == expected_project_id + ) + elif call.get("function_name") == "mcp__kernel__manage_browsers": + browser_arguments_valid = ( + browser_arguments_valid and _manage_browsers_arguments_valid(call) + ) + + native_calls_present = all(calls_by_name[name] for name in REQUIRED_TOOLS) + observations_valid = bool(calls) and not ( + missing_observations or duplicate_observations or error_observations + ) + return { + "native_calls_present": native_calls_present, + "observations_valid": observations_valid, + "context_scope_valid": context_scope_valid + and bool(calls_by_name["mcp__kernel__get_connection_context"]), + "manage_browsers_arguments_valid": browser_arguments_valid + and bool(calls_by_name["mcp__kernel__manage_browsers"]), + "missing_observations": missing_observations, + "duplicate_observations": duplicate_observations, + "error_observations": error_observations, + "tool_calls": [ + { + "tool_call_id": call.get("tool_call_id"), + "name": call.get("function_name"), + "arguments": call.get("arguments"), + } + for call in calls + ], + } def main() -> int: VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - requests = read_requests(LOGS_DIR / "kernel-mcp/requests.jsonl") report = read_json(LOGS_DIR / "artifacts/agent-report.json") trajectory = read_json(LOGS_DIR / "agent/trajectory.json") manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") - - context_call = successful_call(requests, "get_connection_context") - browsers_call = successful_call(requests, "manage_browsers") expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") - report_matches = bool( - report - and report.get("get_connection_context_succeeded") is True - and report.get("manage_browsers_list_succeeded") is True - and report.get("connection_scope_kind") == "project" - and report.get("project_id") == expected_project_id - ) + atif = validate_trajectory(trajectory, expected_project_id) source_sha_matches = bool( manifest and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") ) - trajectory_names = trajectory_tool_names(trajectory) - trajectory_has_calls = any( - name.endswith("get_connection_context") for name in trajectory_names - ) and any(name.endswith("manage_browsers") for name in trajectory_names) hypeman_identity_present = bool( manifest and manifest.get("hypeman_instance_name") ) checks = { - "get_connection_context": context_call is not None, - "manage_browsers_list": browsers_call is not None, - "agent_report": report_matches, + "native_mcp_calls": atif["native_calls_present"], + "tool_observations": atif["observations_valid"], + "context_scope": atif["context_scope_valid"], + "manage_browsers_arguments": atif["manage_browsers_arguments_valid"], "source_sha": source_sha_matches, "hypeman_identity": hypeman_identity_present, - "trajectory": trajectory_has_calls, "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), } reward = 1.0 if all(checks.values()) else 0.0 - tool_calls = [ - { - "name": call["tool_name"], - "success": call["success"], - "duration_ms": call["duration_ms"], - "http_status": call["http_status"], - } - for call in (context_call, browsers_call) - if call is not None - ] result = { "reward": reward, "checks": checks, - "tool_calls": tool_calls, + "atif": atif, "agent_report": report, "trajectory": { "present": trajectory is not None, "schema_version": (trajectory or {}).get("schema_version"), "agent": (trajectory or {}).get("agent"), - "tool_names": trajectory_names, }, "run_manifest": manifest, } @@ -126,15 +202,7 @@ def main() -> int: (VERIFIER_DIR / "reward.txt").write_text(str(reward)) (VERIFIER_DIR / "reward.json").write_text( json.dumps( - { - "reward": reward, - "get_connection_context": float(context_call is not None), - "manage_browsers_list": float(browsers_call is not None), - "agent_report": float(report_matches), - "source_sha": float(source_sha_matches), - "hypeman_identity": float(hypeman_identity_present), - "trajectory": float(trajectory_has_calls), - }, + {"reward": reward, **{name: float(value) for name, value in checks.items()}}, indent=2, ) ) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index a96e3fe..b2aaa0f 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -27,7 +27,6 @@ RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ && install -m 0755 benchmarks/harbor/bin/verify-smoke.py /usr/local/bin/verify-kernel-mcp-smoke \ - && install -m 0644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs /usr/local/lib/mcp-telemetry-proxy.mjs \ && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA ENV NEXT_TELEMETRY_DISABLED=1 diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml index d2b12cb..055bd16 100644 --- a/benchmarks/harbor/smoke/task.toml +++ b/benchmarks/harbor/smoke/task.toml @@ -44,7 +44,7 @@ timeout_sec = 300.0 timeout_sec = 60.0 [steps.healthcheck] -command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp && curl -sS -o /dev/null http://127.0.0.1:3003/mcp" +command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp" interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 1.0 From b193ca089e0ea7fba92560e60c53e089c5f64649 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:50:09 +0000 Subject: [PATCH 05/22] Support Codex ATIF observations --- .../trajectory-codex-observation.json | 37 +++++++++++++++++++ benchmarks/harbor/bin/test_verify_smoke.py | 10 +++++ benchmarks/harbor/bin/verify-smoke.py | 21 +++++++++-- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json diff --git a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json new file mode 100644 index 0000000..d280135 --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "[{'type': 'text', 'text': '{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}'}]" + }, + { + "source_call_id": "browsers-1", + "content": "[{'type': 'text', 'text': '{\"items\":[],\"has_more\":false}'}]" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py index 18d53be..707468b 100644 --- a/benchmarks/harbor/bin/test_verify_smoke.py +++ b/benchmarks/harbor/bin/test_verify_smoke.py @@ -28,6 +28,16 @@ def test_accepts_native_calls_with_paired_observations(self) -> None: self.assertTrue(proof["context_scope_valid"]) self.assertTrue(proof["manage_browsers_arguments_valid"]) + def test_accepts_codex_serialized_observations(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-codex-observation.json"), "project-123" + ) + + self.assertTrue(proof["native_calls_present"]) + self.assertTrue(proof["observations_valid"]) + self.assertTrue(proof["context_scope_valid"]) + self.assertTrue(proof["manage_browsers_arguments_valid"]) + def test_rejects_missing_observation(self) -> None: proof = verify_smoke.validate_trajectory( load_fixture("trajectory-missing-observation.json"), "project-123" diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py index 5b0de04..a3a9e85 100755 --- a/benchmarks/harbor/bin/verify-smoke.py +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from __future__ import annotations +import ast import json import os from pathlib import Path @@ -24,16 +25,27 @@ def read_json(path: Path) -> dict[str, Any] | None: def _decode_tool_result_content(content: Any) -> Any: value = content - for _ in range(4): + for _ in range(6): if isinstance(value, str): try: value = json.loads(value) except json.JSONDecodeError: - return value + try: + value = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value continue if isinstance(value, dict) and value.get("type") == "text": value = value.get("text") continue + if ( + isinstance(value, list) + and len(value) == 1 + and isinstance(value[0], dict) + and value[0].get("type") == "text" + ): + value = value[0].get("text") + continue return value return value @@ -119,13 +131,14 @@ def validate_trajectory( continue result = results[0] decoded = _decode_tool_result_content(result.get("content")) - if _contains_error(decoded) or _contains_error(result.get("extra")): + if _contains_error(decoded) or _contains_error(result): error_observations.append(call_id) continue if call.get("function_name") == "mcp__kernel__get_connection_context": scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None context_scope_valid = context_scope_valid and ( - isinstance(scope, dict) + bool(expected_project_id) + and isinstance(scope, dict) and scope.get("kind") == "project" and scope.get("project_id") == expected_project_id ) From 2c8007d0b7ff8f7cfe6fc4fe4604ad9196b7a6e2 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:57:05 +0000 Subject: [PATCH 06/22] Pin harbor-hypeman benchmark release --- benchmarks/harbor/README.md | 14 +++++++++++--- benchmarks/harbor/run-smoke.sh | 17 ++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index bc08cb8..11c73d6 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -4,14 +4,22 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv ## Requirements -- Harbor 0.21.0 with `harbor_hypeman:HypemanEnvironment` -- `harbor-hypeman` with existing Hypeman image-reference support -- Hypeman CLI and credentials +- Harbor 0.21.0 +- `harbor-hypeman` 0.1.1 +- [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` - `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex +`run-smoke.sh` launches the pinned Harbor packages through `uvx`. To install the same versions as a persistent tool instead: + +```bash +uv tool install 'harbor==0.21.0' --with 'harbor-hypeman==0.1.1' +``` + +Set `HARBOR_BIN` only when intentionally testing a different Harbor installation. + ## Build the image ```bash diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index d4c447c..1618978 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -46,13 +46,16 @@ case "$agent" in esac if [[ -n "${HARBOR_BIN:-}" ]]; then - harbor_bin=$HARBOR_BIN -elif command -v harbor >/dev/null 2>&1; then - harbor_bin=$(command -v harbor) -elif [[ -x "$repo_root/../harbor-hypeman/.venv/bin/harbor" ]]; then - harbor_bin="$repo_root/../harbor-hypeman/.venv/bin/harbor" + harbor_command=("$HARBOR_BIN") +elif command -v uvx >/dev/null 2>&1; then + harbor_command=( + uvx + --from "harbor==0.21.0" + --with "harbor-hypeman==0.1.1" + harbor + ) else - echo "Harbor CLI not found; set HARBOR_BIN" >&2 + echo "uvx not found; install uv or set HARBOR_BIN" >&2 exit 1 fi @@ -81,7 +84,7 @@ chmod 0600 "$runtime_env" mkdir -p "$jobs_dir" timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-10m}" \ - "$harbor_bin" run \ + "${harbor_command[@]}" run \ --path "$runtime_task" \ --agent "$agent" \ --model "$model" \ From 3413edf03bf6a11dbaa6ae1d106ef367ee6b899f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:04:51 +0000 Subject: [PATCH 07/22] Forward Codex credentials to benchmark --- benchmarks/harbor/run-smoke.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 1618978..7e1571a 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -79,6 +79,7 @@ ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" From 535f60971dc1646670ba44eb176d847e9743ea5e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:27:04 +0000 Subject: [PATCH 08/22] Add ClawBench Kernel MCP control arm --- README.md | 2 +- benchmarks/harbor/README.md | 20 ++ benchmarks/harbor/bin/start-kernel-mcp-server | 10 + .../harbor/clawbench/prepare-control.py | 175 +++++++++++++ benchmarks/harbor/clawbench/run-control.sh | 131 ++++++++++ benchmarks/harbor/clawbench/test_control.py | 184 +++++++++++++ benchmarks/harbor/clawbench/verify-control.py | 245 ++++++++++++++++++ benchmarks/harbor/image/Dockerfile | 2 + src/lib/mcp/register.test.ts | 31 +++ src/lib/mcp/register.ts | 41 ++- 10 files changed, 837 insertions(+), 4 deletions(-) create mode 100755 benchmarks/harbor/clawbench/prepare-control.py create mode 100755 benchmarks/harbor/clawbench/run-control.sh create mode 100644 benchmarks/harbor/clawbench/test_control.py create mode 100755 benchmarks/harbor/clawbench/verify-control.py diff --git a/README.md b/README.md index f9bd672..06a6420 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ Each Kernel feature has a single `manage_*` tool with an `action` parameter, kee One additional Managed Auth helper (`begin_auth_login`) is marked app-only (`_meta.ui.visibility: ["app"]`); it refuses to execute on hosts that do not declare MCP Apps support. The App forwards the server-issued signed flow checkpoint to the shared `manage_auth_connections` `wait` action, so flow identity and terminal-state decisions stay on the server. -Self-hosted deployments can hide sensitive tool families by setting `KERNEL_MCP_DISABLED_TOOLSETS` to a comma-separated list. For example, `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` prevents `manage_api_keys` from being registered. +Self-hosted deployments can select tool families with `KERNEL_MCP_ENABLED_TOOLSETS` or hide them with `KERNEL_MCP_DISABLED_TOOLSETS`. Both accept comma- or space-separated toolset names and standalone aliases. For example, `KERNEL_MCP_ENABLED_TOOLSETS="playwright computer"` exposes browser-control tools without browser lifecycle or managed-auth tools, while `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` only removes `manage_api_keys`. `get_connection_context` remains available in either mode. Call `get_connection_context` before deciding whether to create or select a project. Its canonical `connection_scope` reports whether the connection is organization-wide or fixed to a project. Project-scoped tools advertise an optional `project` (name or ID) and a deprecated `project_id`: organization-wide connections may omit them to preserve organization-wide reads and API default-project behavior, while fixed-project connections may omit them or pass the matching project. Project resources use project-qualified `kernel://orgs/{organizationId}/projects/{projectId}/...` URIs. Authorization remains enforced by the Kernel API; selecting a project never grants access to it. diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 11c73d6..5ff8de6 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -54,3 +54,23 @@ The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful - numeric Harbor rewards plus detailed `smoke-result.json` The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. + +## Run the ClawBench Kernel MCP arm + +The ClawBench arm starts from the Kernel-backed Harbor task produced by `clawbench-harbor-adapt`, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. + +```bash +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run-control.sh claude-code \ + v2-1134-chapter-finder-redcross +``` + +The ClawBench checkout must contain commit `6efb04e`, from `kernel/ClawBench` PR #1. The generated task: + +- exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` +- disables browser lifecycle and managed-auth toolsets +- instructs the agent to read `./my-info/kernel_browser.json` and use that session ID +- instructs account tasks to use the supplied PurelyMail credentials instead of managed auth +- verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion + +Outputs use the normal Harbor job directory and add `kernel-mcp-control-result.json`, Kernel MCP logs, source manifests, and same-session metrics to the ClawBench verifier artifacts. diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index 08e7601..727c95d 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -45,8 +45,18 @@ import platform from datetime import datetime, timezone from pathlib import Path +browser_path = Path("/my-info/kernel_browser.json") +try: + browser = json.loads(browser_path.read_text()) +except (OSError, json.JSONDecodeError): + browser = {} + manifest = { "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "clawbench_source_sha": os.environ.get("CLAWBENCH_SOURCE_SHA", ""), + "browser_session_id": browser.get("session_id"), + "enabled_toolsets": os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", ""), + "disabled_toolsets": os.environ.get("KERNEL_MCP_DISABLED_TOOLSETS", ""), "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), "sandbox_hostname": platform.node(), diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py new file mode 100755 index 0000000..50b58b6 --- /dev/null +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + +ENABLED_TOOLSETS = "playwright computer" + + +def _drop_mcp_servers(task_toml: str) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + dropping = False + for line in lines: + if line.strip() == "[[environment.mcp_servers]]": + dropping = True + continue + if dropping and line.startswith("["): + dropping = False + if not dropping: + output.append(line) + return "\n".join(output).rstrip() + "\n" + + +def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_sha: str) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + inserted_image = False + inserted_env = False + for line in lines: + output.append(line) + if line.strip() == "[environment]": + output.append(f"docker_image = {json.dumps(image)}") + inserted_image = True + elif line.strip() == "[environment.env]": + output.extend( + [ + f"KERNEL_MCP_BENCHMARK_IMAGE = {json.dumps(image)}", + f"KERNEL_MCP_SOURCE_SHA = {json.dumps(server_sha)}", + f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", + f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", + 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', + 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + ] + ) + inserted_env = True + if not inserted_image or not inserted_env: + raise ValueError("generated task is missing Harbor environment sections") + output.extend( + [ + "", + "[[environment.mcp_servers]]", + 'name = "kernel"', + 'transport = "stdio"', + 'command = "/usr/local/bin/kernel-mcp-local"', + "args = []", + ] + ) + return "\n".join(output).rstrip() + "\n" + + +def _patch_setup(setup: str) -> str: + install = """install_clawbench_runtime() { + mkdir -p /app/src + rm -rf /app/src/runtime-server /app/src/chrome-extension /app/src/shared /app/src/harbor + cp -a /runtime-server /app/src/runtime-server + cp -a /chrome-extension /app/src/chrome-extension + cp -a /shared /app/src/shared + cp -a /harbor /app/src/harbor + chmod +x /app/src/harbor/*.sh /app/src/harbor/*.py + cd /app/src/runtime-server + UV_PYTHON_PREFERENCE=only-system uv sync --frozen + uv pip install --python .venv/bin/python fpdf2 + cd / +} + +install_clawbench_runtime +""" + marker = "mkdir -p /data /logs/verifier /extra_info\n" + if marker not in setup: + raise ValueError("generated setup script is missing directory initialization") + setup = setup.replace(marker, marker + "\n" + install, 1) + runtime_marker = "/app/src/harbor/start-runtime.sh\n" + if runtime_marker not in setup: + raise ValueError("generated setup script is missing runtime startup") + return setup.replace( + runtime_marker, + runtime_marker + "\nstart-kernel-mcp-server\n", + 1, + ) + + +def _patch_verifier(test_script: str) -> str: + verify_marker = "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + if verify_marker not in test_script: + raise ValueError("generated verifier script is missing ClawBench verification") + return test_script.replace( + verify_marker, + verify_marker + + "mkdir -p /data/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /data/kernel-mcp/\n" + + "/app/src/runtime-server/.venv/bin/python " + + "/app/src/harbor/verify-kernel-mcp-control.py\n", + 1, + ) + + +def _patch_instruction(instruction: str) -> str: + return instruction.rstrip() + """ + +--- +Kernel MCP benchmark arm: +- Call `get_connection_context` once before taking any browser action. +- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. +- Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. +- Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. +- Complete and submit the task through the existing browser, then stop. +""" + + +def transform_task(task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str) -> None: + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.unlink(missing_ok=True) + + task_toml_path = task_dir / "task.toml" + task_toml = _drop_mcp_servers(task_toml_path.read_text()) + task_toml_path.write_text( + _add_environment( + task_toml, + image=image, + server_sha=server_sha, + clawbench_sha=clawbench_sha, + ) + ) + + step_dir = task_dir / "steps" / "run" + setup_path = step_dir / "workdir" / "setup.sh" + setup_path.write_text(_patch_setup(setup_path.read_text())) + setup_path.chmod(0o755) + + test_path = step_dir / "tests" / "test.sh" + test_path.write_text(_patch_verifier(test_path.read_text())) + test_path.chmod(0o755) + + instruction_path = step_dir / "instruction.md" + instruction_path.write_text(_patch_instruction(instruction_path.read_text())) + + verifier_source = Path(__file__).with_name("verify-control.py") + verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-control.py" + shutil.copy2(verifier_source, verifier_target) + verifier_target.chmod(0o755) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert a Kernel-backed ClawBench Harbor task to the Kernel MCP arm") + parser.add_argument("task_dir", type=Path) + parser.add_argument("--image", required=True) + parser.add_argument("--server-sha", required=True) + parser.add_argument("--clawbench-sha", required=True) + args = parser.parse_args() + transform_task( + args.task_dir, + image=args.image, + server_sha=args.server_sha, + clawbench_sha=args.clawbench_sha, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh new file mode 100755 index 0000000..db8a3f2 --- /dev/null +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -0,0 +1,131 @@ +#!/bin/bash +set -euo pipefail + +usage() { + echo "usage: $0 [task-id] [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage +task_id=${2:-v2-1134-chapter-finder-redcross} + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +clawbench_ref=${CLAWBENCH_REF:-6efb04e49efc44f36fa03c8be3bcdb3ef091434f} + +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} +[[ -d "$clawbench_repo/.git" || -f "$clawbench_repo/.git" ]] || { + echo "ClawBench checkout not found at $clawbench_repo" >&2 + exit 1 +} +git -C "$clawbench_repo" merge-base --is-ancestor "$clawbench_ref" HEAD || { + echo "ClawBench checkout must contain $clawbench_ref" >&2 + exit 1 +} + +if [[ -f "$clawbench_repo/.env" ]]; then + set -a + source "$clawbench_repo/.env" + set +a +fi +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" +: "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" +: "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" + +case "$agent" in + claude-code) + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} + version=${CODEX_BENCHMARK_VERSION:-0.120.0} + ;; +esac + +runtime_root=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT + +dataset="$runtime_root/dataset" +uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ + --output-dir "$dataset" \ + --task-ids "$task_id" \ + --browser-runtime kernel \ + --browser-runtime-options '{"stealth": false}' \ + --overwrite + +task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) +[[ -n "$task_dir" ]] || { + echo "ClawBench did not generate task $task_id" >&2 + exit 1 +} + +python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" + +export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY +export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} +export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} +export KERNEL_MCP_BENCHMARK_PROJECT_ID + +cat >"$runtime_env" < ModuleType: + spec = importlib.util.spec_from_file_location(name, HERE / filename) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +prepare = load_module("prepare_control", "prepare-control.py") +verify = load_module("verify_control", "verify-control.py") + + +class PrepareControlTest(unittest.TestCase): + def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: + with tempfile.TemporaryDirectory() as temp: + task = Path(temp) + environment = task / "environment" + step = task / "steps" / "run" + (step / "workdir").mkdir(parents=True) + (step / "tests").mkdir() + (step / "instruction.md").write_text("Complete the browser task.\n") + (environment / "harbor").mkdir(parents=True) + (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") + (task / "task.toml").write_text( + """[environment] +workdir = "/" + +[environment.env] +KERNEL_API_KEY = "${KERNEL_API_KEY}" + +[[steps]] +name = "run" + +[[environment.mcp_servers]] +name = "playwright" +transport = "stdio" +command = "npx" +args = ["-y", "@playwright/mcp@0.0.79"] +""" + ) + (step / "workdir" / "setup.sh").write_text( + "#!/bin/bash\nmkdir -p /data /logs/verifier /extra_info\n" + "/app/src/harbor/start-runtime.sh\n" + ) + (step / "tests" / "test.sh").write_text( + "#!/bin/bash\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + ) + + prepare.transform_task( + task, + image="docker.io/builds/image:latest", + server_sha="server-sha", + clawbench_sha="clawbench-sha", + ) + + task_toml = (task / "task.toml").read_text() + self.assertFalse((environment / "Dockerfile").exists()) + self.assertIn('docker_image = "docker.io/builds/image:latest"', task_toml) + self.assertIn('name = "kernel"', task_toml) + self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) + self.assertNotIn("@playwright/mcp", task_toml) + self.assertIn( + 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml + ) + self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + + setup = (step / "workdir" / "setup.sh").read_text() + self.assertIn("install_clawbench_runtime", setup) + self.assertIn("start-kernel-mcp-server", setup) + test_script = (step / "tests" / "test.sh").read_text() + self.assertIn("verify-kernel-mcp-control.py", test_script) + self.assertIn("/data/kernel-mcp", test_script) + instruction = (step / "instruction.md").read_text() + self.assertIn("existing `session_id`", instruction) + self.assertIn("PurelyMail-backed credentials", instruction) + self.assertIn("Do not use Kernel managed auth", instruction) + self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) + + +class VerifyControlTest(unittest.TestCase): + def trajectory(self, session_id: str = "session-123") -> dict: + return { + "steps": [ + { + "tool_calls": [ + { + "tool_call_id": "context-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {}, + }, + { + "tool_call_id": "playwright-1", + "function_name": "mcp__kernel__execute_playwright_code", + "arguments": { + "session_id": session_id, + "code": "await page.goto('https://example.com')", + }, + }, + ], + "observation": { + "results": [ + { + "source_call_id": "context-1", + "content": { + "connection_scope": { + "kind": "project", + "project_id": "project-123", + } + }, + }, + { + "source_call_id": "playwright-1", + "content": [{"type": "text", "text": "{\"ok\": true}"}], + }, + ] + }, + } + ] + } + + def test_accepts_successful_calls_on_precreated_session(self) -> None: + result = verify.validate_control( + self.trajectory(), + expected_session_id="session-123", + expected_project_id="project-123", + ) + for key in ( + "context_called", + "browser_control_called", + "observations_valid", + "context_scope_valid", + "same_session", + "no_playwright_mcp", + "no_forbidden_kernel_tools", + ): + self.assertTrue(result[key], key) + + def test_rejects_another_session(self) -> None: + result = verify.validate_control( + self.trajectory("session-other"), + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["same_session"]) + + def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].extend( + [ + { + "tool_call_id": "direct-playwright", + "function_name": "mcp__playwright__browser_navigate", + "arguments": {}, + }, + { + "tool_call_id": "browser-list", + "function_name": "mcp__kernel__manage_browsers", + "arguments": {"action": "list"}, + }, + ] + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_playwright_mcp"]) + self.assertFalse(result["no_forbidden_kernel_tools"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py new file mode 100755 index 0000000..681dac8 --- /dev/null +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import ast +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" +CONTEXT_TOOL = "mcp__kernel__get_connection_context" +BROWSER_TOOLS = { + "mcp__kernel__execute_playwright_code", + "mcp__kernel__computer_action", +} +FORBIDDEN_KERNEL_TOOLS = { + "mcp__kernel__manage_browsers", + "mcp__kernel__manage_auth_connections", + "mcp__kernel__open_auth_login", +} + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _decode_content(content: Any) -> Any: + value = content + for _ in range(6): + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + try: + value = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value + continue + if isinstance(value, dict) and value.get("type") == "text": + value = value.get("text") + continue + if ( + isinstance(value, list) + and len(value) == 1 + and isinstance(value[0], dict) + and value[0].get("type") == "text" + ): + value = value[0].get("text") + continue + return value + return value + + +def _contains_error(value: Any) -> bool: + if isinstance(value, dict): + if value.get("is_error") is True or value.get("isError") is True: + return True + if value.get("error") not in (None, False, ""): + return True + return any(_contains_error(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error(item) for item in value) + if isinstance(value, str): + return value.lstrip().lower().startswith(("[error]", "error:", "error in ")) + return False + + +def _calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + calls.extend(call for call in step.get("tool_calls") or [] if isinstance(call, dict)) + return calls + + +def _results(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + results: dict[str, list[dict[str, Any]]] = {} + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + observation = step.get("observation") + if not isinstance(observation, dict): + continue + for result in observation.get("results") or []: + if not isinstance(result, dict): + continue + call_id = result.get("source_call_id") + if isinstance(call_id, str): + results.setdefault(call_id, []).append(result) + return results + + +def validate_control( + trajectory: dict[str, Any] | None, + *, + expected_session_id: str, + expected_project_id: str, +) -> dict[str, Any]: + trajectory = trajectory or {} + calls = _calls(trajectory) + result_map = _results(trajectory) + kernel_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__kernel__") + ] + context_calls = [call for call in kernel_calls if call.get("function_name") == CONTEXT_TOOL] + browser_calls = [call for call in kernel_calls if call.get("function_name") in BROWSER_TOOLS] + playwright_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__playwright__") + ] + forbidden_calls = [ + call for call in kernel_calls if call.get("function_name") in FORBIDDEN_KERNEL_TOOLS + ] + + missing_observations: list[Any] = [] + duplicate_observations: list[Any] = [] + error_observations: list[Any] = [] + context_scope_valid = bool(expected_project_id) + same_session = bool(expected_session_id and browser_calls) + + for call in context_calls + browser_calls: + call_id = call.get("tool_call_id") + observations = result_map.get(call_id, []) if isinstance(call_id, str) else [] + if not observations: + missing_observations.append(call_id) + continue + if len(observations) != 1: + duplicate_observations.append(call_id) + continue + decoded = _decode_content(observations[0].get("content")) + if _contains_error(decoded) or _contains_error(observations[0]): + error_observations.append(call_id) + continue + if call.get("function_name") == CONTEXT_TOOL: + scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None + context_scope_valid = context_scope_valid and ( + isinstance(scope, dict) + and scope.get("kind") == "project" + and scope.get("project_id") == expected_project_id + ) + elif call.get("function_name") in BROWSER_TOOLS: + arguments = call.get("arguments") + same_session = same_session and ( + isinstance(arguments, dict) + and arguments.get("session_id") == expected_session_id + ) + + observations_valid = bool(context_calls and browser_calls) and not ( + missing_observations or duplicate_observations or error_observations + ) + return { + "context_called": bool(context_calls), + "browser_control_called": bool(browser_calls), + "observations_valid": observations_valid, + "context_scope_valid": context_scope_valid and bool(context_calls), + "same_session": same_session, + "no_playwright_mcp": not playwright_calls, + "no_forbidden_kernel_tools": not forbidden_calls, + "missing_observations": missing_observations, + "duplicate_observations": duplicate_observations, + "error_observations": error_observations, + "kernel_tool_calls": [ + { + "tool_call_id": call.get("tool_call_id"), + "name": call.get("function_name"), + "arguments": call.get("arguments"), + } + for call in kernel_calls + ], + } + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + trajectory = read_json(LOGS_DIR / "agent" / "trajectory.json") + browser = read_json(Path("/my-info/kernel_browser.json")) + lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) + manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") + clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") + reward_path = VERIFIER_DIR / "reward.json" + reward_metrics = read_json(reward_path) or {} + + session_id = str((browser or {}).get("session_id") or "") + expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + atif = validate_control( + trajectory, + expected_session_id=session_id, + expected_project_id=expected_project_id, + ) + checks = { + "kernel_mcp_context": atif["context_called"], + "kernel_mcp_browser_control": atif["browser_control_called"], + "kernel_mcp_observations": atif["observations_valid"], + "kernel_mcp_project_scope": atif["context_scope_valid"], + "kernel_mcp_same_session": atif["same_session"], + "no_playwright_mcp": atif["no_playwright_mcp"], + "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], + "kernel_mcp_source_sha": bool( + manifest + and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") + ), + "kernel_mcp_manifest_session": bool( + manifest and manifest.get("browser_session_id") == session_id + ), + "kernel_mcp_toolset_allowlist": bool( + manifest + and set(str(manifest.get("enabled_toolsets", "")).split()) + == {"playwright", "computer"} + ), + "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), + "browser_deleted": bool( + lifecycle + and lifecycle.get("status") == "deleted" + and lifecycle.get("deletion_verified") is True + ), + "clawbench_intercepted": bool( + reward_metrics.get("intercepted") == 1 + or (clawbench_result or {}).get("intercepted") is True + ), + } + infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") + checks["infra_ok"] = infra_ok + + reward_metrics.update({name: float(value) for name, value in checks.items()}) + reward_path.write_text(json.dumps(reward_metrics, indent=2)) + result = { + "checks": checks, + "session_id": session_id, + "expected_project_id": expected_project_id, + "atif": atif, + "run_manifest": manifest, + "browser_lifecycle": lifecycle, + "clawbench_result": clawbench_result, + } + (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index b2aaa0f..be85f3a 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -12,6 +12,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && npm install --global bun@1.3.3 +COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv + WORKDIR /opt/kernel-mcp-server COPY package.json bun.lock ./ diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 52061de..a6b0c94 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -83,6 +83,37 @@ describe("MCP Apps additive registration", () => { }); }); +describe("MCP toolset allowlist", () => { + test("keeps connection context and only the selected browser controls", () => { + const previousEnabled = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + const previousDisabled = process.env.KERNEL_MCP_DISABLED_TOOLSETS; + process.env.KERNEL_MCP_ENABLED_TOOLSETS = + "execute_playwright_code computer_action"; + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + try { + const registration = captureRegistration(false); + expect(registration.legacyTools).toEqual([ + "get_connection_context", + "computer_action", + "execute_playwright_code", + ]); + expect(registration.appTools).toEqual([]); + expect(registration.resources).toEqual([]); + } finally { + if (previousEnabled === undefined) { + delete process.env.KERNEL_MCP_ENABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_ENABLED_TOOLSETS = previousEnabled; + } + if (previousDisabled === undefined) { + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_DISABLED_TOOLSETS = previousDisabled; + } + } + }); +}); + describe("project selection registration", () => { const projectScopedTools = [ "manage_profiles", diff --git a/src/lib/mcp/register.ts b/src/lib/mcp/register.ts index 59db2cb..712be6c 100644 --- a/src/lib/mcp/register.ts +++ b/src/lib/mcp/register.ts @@ -90,6 +90,33 @@ function normalizeMcpToolset(value: string): McpToolset | undefined { return undefined; } +function enabledMcpToolsetsFromEnv() { + const raw = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + if (!raw?.trim()) return undefined; + + const enabled = new Set(); + const unknown: string[] = []; + for (const value of raw.split(/[,\s]+/)) { + const token = value.trim().toLowerCase(); + if (!token || token === "none") continue; + if (token === "all") return new Set(mcpToolsets); + + const toolset = normalizeMcpToolset(token); + if (toolset) { + enabled.add(toolset); + } else { + unknown.push(value); + } + } + + if (unknown.length > 0) { + throw new Error( + `Unknown KERNEL_MCP_ENABLED_TOOLSETS value(s): ${unknown.join(", ")}. Supported toolsets: ${mcpToolsets.join(", ")}.`, + ); + } + return enabled; +} + function disabledMcpToolsetsFromEnv() { const raw = process.env.KERNEL_MCP_DISABLED_TOOLSETS; if (!raw?.trim()) return new Set(); @@ -126,10 +153,14 @@ function disabledMcpToolsetsFromEnv() { } function toolsetEnabled( + enabledToolsets: Set | undefined, disabledToolsets: Set, toolset: McpToolset, ) { - return !disabledToolsets.has(toolset); + return ( + (enabledToolsets === undefined || enabledToolsets.has(toolset)) && + !disabledToolsets.has(toolset) + ); } export function registerMcpCapabilities( @@ -139,6 +170,7 @@ export function registerMcpCapabilities( dependencies = defaultMcpDependencies, }: McpRegistrationOptions = {}, ) { + const enabledToolsets = enabledMcpToolsetsFromEnv(); const disabledToolsets = disabledMcpToolsetsFromEnv(); registerKernelPrompts(server); @@ -147,7 +179,7 @@ export function registerMcpCapabilities( registerConnectionContextTool(server); for (const [toolset, registerToolset] of mcpToolRegistrations) { - if (toolsetEnabled(disabledToolsets, toolset)) { + if (toolsetEnabled(enabledToolsets, disabledToolsets, toolset)) { registerToolset(server, dependencies); } } @@ -155,7 +187,10 @@ export function registerMcpCapabilities( // Managed Auth remains fully programmatic for every client. MCP Apps support // adds one interactive launcher (plus its app-only implementation tools and // resource) without replacing or narrowing manage_auth_connections. - if (mcpApps && toolsetEnabled(disabledToolsets, "auth_connections")) { + if ( + mcpApps && + toolsetEnabled(enabledToolsets, disabledToolsets, "auth_connections") + ) { registerAuthLoginApp(server); } } From 471dc58e081a3ecb414118521481bab5ae1f7c6e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:38:00 +0000 Subject: [PATCH 09/22] Fix benchmark runtime configuration --- benchmarks/harbor/bin/start-kernel-mcp-server | 3 +++ benchmarks/harbor/clawbench/run-control.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index 727c95d..2096588 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -14,6 +14,9 @@ chmod 0600 "$key_dir/api-key" redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ --logfile "$log_dir/redis.log" --dir /tmp +export CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-sk_test_kernel_mcp_benchmark_local_only} +export NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ} + cd /opt/kernel-mcp-server nohup ./node_modules/.bin/next start -p 3002 \ >"$log_dir/server.stdout.log" \ diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index db8a3f2..69ce746 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -105,6 +105,7 @@ CLAWBENCH_JUDGE_MODEL=${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro} CLAWBENCH_JUDGE_API_TYPE=${CLAWBENCH_JUDGE_API_TYPE:-openai-completions} ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} +ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} OPENAI_API_KEY=${OPENAI_API_KEY:-} From 1ef8cd8bbb9e3b2f648f16e52b9af85fa0b41025 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:50:27 +0000 Subject: [PATCH 10/22] Make benchmark MCP startup deterministic --- benchmarks/harbor/bin/kernel-mcp-local | 2 +- benchmarks/harbor/clawbench/prepare-control.py | 10 ++++++++-- benchmarks/harbor/clawbench/run-control.sh | 2 +- benchmarks/harbor/clawbench/test_control.py | 9 +++++++-- benchmarks/harbor/image/Dockerfile | 2 +- benchmarks/harbor/run-smoke.sh | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local index 5b79a8e..9cb8c06 100755 --- a/benchmarks/harbor/bin/kernel-mcp-local +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -8,6 +8,6 @@ if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then fi : "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" -exec npx -y mcp-remote@0.1.38 \ +exec mcp-remote \ http://127.0.0.1:3002/mcp \ --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 50b58b6..a57dd50 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -43,6 +43,7 @@ def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_s f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) inserted_env = True @@ -99,8 +100,8 @@ def _patch_verifier(test_script: str) -> str: return test_script.replace( verify_marker, verify_marker - + "mkdir -p /data/kernel-mcp\n" - + "cp -a /logs/kernel-mcp/. /data/kernel-mcp/\n" + + "mkdir -p /logs/verifier/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /logs/verifier/kernel-mcp/\n" + "/app/src/runtime-server/.venv/bin/python " + "/app/src/harbor/verify-kernel-mcp-control.py\n", 1, @@ -108,10 +109,15 @@ def _patch_verifier(test_script: str) -> str: def _patch_instruction(instruction: str) -> str: + instruction = instruction.replace( + "Use only Playwright MCP browser tools plus reading files", + "Use only Kernel MCP browser-control tools plus reading files", + ) return instruction.rstrip() + """ --- Kernel MCP benchmark arm: +- Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. - Call `get_connection_context` once before taking any browser action. - Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 69ce746..8bf985d 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -107,7 +107,7 @@ ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} -CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-false} OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index d60a0d1..cf374c0 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -29,7 +29,9 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: step = task / "steps" / "run" (step / "workdir").mkdir(parents=True) (step / "tests").mkdir() - (step / "instruction.md").write_text("Complete the browser task.\n") + (step / "instruction.md").write_text( + "Use only Playwright MCP browser tools plus reading files under ./my-info/.\n" + ) (environment / "harbor").mkdir(parents=True) (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") (task / "task.toml").write_text( @@ -75,15 +77,18 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml ) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) setup = (step / "workdir" / "setup.sh").read_text() self.assertIn("install_clawbench_runtime", setup) self.assertIn("start-kernel-mcp-server", setup) test_script = (step / "tests" / "test.sh").read_text() self.assertIn("verify-kernel-mcp-control.py", test_script) - self.assertIn("/data/kernel-mcp", test_script) + self.assertIn("/logs/verifier/kernel-mcp", test_script) instruction = (step / "instruction.md").read_text() + self.assertIn("WaitForMcpServers", instruction) self.assertIn("existing `session_id`", instruction) + self.assertIn("Use only Kernel MCP browser-control tools", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index be85f3a..dfc0d5b 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update \ python3 \ redis-server \ && rm -rf /var/lib/apt/lists/* \ - && npm install --global bun@1.3.3 + && npm install --global bun@1.3.3 mcp-remote@0.1.38 COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 7e1571a..c011da6 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -78,7 +78,7 @@ KERNEL_PROJECT=${KERNEL_PROJECT:-} ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} -CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-false} OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" From 4128d8525ff71ff50b70bc412bed07992b02a040 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:11:01 +0000 Subject: [PATCH 11/22] Verify ClawBench same-session execution --- .../harbor/clawbench/prepare-control.py | 1 + benchmarks/harbor/clawbench/test_control.py | 14 ++++++++ benchmarks/harbor/clawbench/verify-control.py | 36 ++++++++++++++----- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index a57dd50..98ef4b2 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -122,6 +122,7 @@ def _patch_instruction(instruction: str) -> str: - Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. - Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. - Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. - Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. - Complete and submit the task through the existing browser, then stop. diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index cf374c0..f2f84f8 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -91,6 +91,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: self.assertIn("Use only Kernel MCP browser-control tools", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) + self.assertIn("Do not call `fetch`", instruction) self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) @@ -149,6 +150,7 @@ def test_accepts_successful_calls_on_precreated_session(self) -> None: "same_session", "no_playwright_mcp", "no_forbidden_kernel_tools", + "no_direct_http_automation", ): self.assertTrue(result[key], key) @@ -160,6 +162,18 @@ def test_rejects_another_session(self) -> None: ) self.assertFalse(result["same_session"]) + def test_rejects_direct_http_inside_playwright_code(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"][1]["arguments"]["code"] = ( + "return await page.evaluate(() => fetch('/api'))" + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_direct_http_automation"]) + def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: trajectory = self.trajectory() trajectory["steps"][0]["tool_calls"].extend( diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index 681dac8..395133d 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -4,6 +4,7 @@ import ast import json import os +import re from pathlib import Path from typing import Any @@ -121,7 +122,13 @@ def validate_control( duplicate_observations: list[Any] = [] error_observations: list[Any] = [] context_scope_valid = bool(expected_project_id) - same_session = bool(expected_session_id and browser_calls) + same_session = bool(expected_session_id and browser_calls) and all( + isinstance(call.get("arguments"), dict) + and call["arguments"].get("session_id") == expected_session_id + for call in browser_calls + ) + successful_context_calls = 0 + successful_browser_calls = 0 for call in context_calls + browser_calls: call_id = call.get("tool_call_id") @@ -143,16 +150,26 @@ def validate_control( and scope.get("kind") == "project" and scope.get("project_id") == expected_project_id ) + successful_context_calls += 1 elif call.get("function_name") in BROWSER_TOOLS: - arguments = call.get("arguments") - same_session = same_session and ( - isinstance(arguments, dict) - and arguments.get("session_id") == expected_session_id - ) + successful_browser_calls += 1 - observations_valid = bool(context_calls and browser_calls) and not ( - missing_observations or duplicate_observations or error_observations + observations_valid = ( + successful_context_calls > 0 + and successful_browser_calls > 0 + and not (missing_observations or duplicate_observations) ) + direct_http_patterns = re.compile( + r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", + re.IGNORECASE, + ) + direct_http_calls = [ + call + for call in browser_calls + if call.get("function_name") == "mcp__kernel__execute_playwright_code" + and isinstance(call.get("arguments"), dict) + and direct_http_patterns.search(str(call["arguments"].get("code", ""))) + ] return { "context_called": bool(context_calls), "browser_control_called": bool(browser_calls), @@ -161,9 +178,11 @@ def validate_control( "same_session": same_session, "no_playwright_mcp": not playwright_calls, "no_forbidden_kernel_tools": not forbidden_calls, + "no_direct_http_automation": not direct_http_calls, "missing_observations": missing_observations, "duplicate_observations": duplicate_observations, "error_observations": error_observations, + "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], "kernel_tool_calls": [ { "tool_call_id": call.get("tool_call_id"), @@ -200,6 +219,7 @@ def main() -> int: "kernel_mcp_same_session": atif["same_session"], "no_playwright_mcp": atif["no_playwright_mcp"], "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], + "no_direct_http_automation": atif["no_direct_http_automation"], "kernel_mcp_source_sha": bool( manifest and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") From 2b1f66dc9ff95afda2ecbecd4aafa98e3d66ae58 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:14:52 +0000 Subject: [PATCH 12/22] Forward benchmark API base URL --- benchmarks/harbor/clawbench/prepare-control.py | 2 +- benchmarks/harbor/clawbench/test_control.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 98ef4b2..040a9d5 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -42,7 +42,7 @@ def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_s f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', - 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index f2f84f8..4f1bf82 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -77,6 +77,8 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml ) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) + self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) setup = (step / "workdir" / "setup.sh").read_text() From 126a24b35b20a6b0bccd14ea75a4f7a6d17fedce Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:21:45 +0000 Subject: [PATCH 13/22] Enable stealth for ClawBench control --- benchmarks/harbor/clawbench/run-control.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 8bf985d..61dfd12 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -73,7 +73,7 @@ uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ --output-dir "$dataset" \ --task-ids "$task_id" \ --browser-runtime kernel \ - --browser-runtime-options '{"stealth": false}' \ + --browser-runtime-options '{"stealth": true}' \ --overwrite task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) From 2bd3de512e1770fa3ec966b1d29814f114e087ef Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:22:59 +0000 Subject: [PATCH 14/22] Pin corrected ClawBench evaluator --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 5ff8de6..e5bd2a4 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `6efb04e`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `bf6d1ff`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 61dfd12..784a5c8 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-6efb04e49efc44f36fa03c8be3bcdb3ef091434f} +clawbench_ref=${CLAWBENCH_REF:-bf6d1ff822c80c3cbb086208955b78fe7c9e9e9d} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From 4ee65f78b3d3990d88e2ed920c3568cbc0221c0a Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:17:57 +0000 Subject: [PATCH 15/22] Pin stop-aware ClawBench runtime --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index e5bd2a4..6c442b3 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `bf6d1ff`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `4f39b26`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 784a5c8..5427ac6 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-bf6d1ff822c80c3cbb086208955b78fe7c9e9e9d} +clawbench_ref=${CLAWBENCH_REF:-4f39b269abaab26cb886b643c0cfe6dde1b78698} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From b7b97f13be228a6d0642ed39b84e4810509619a5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:24:21 +0000 Subject: [PATCH 16/22] Accept intercepted terminal tool calls --- benchmarks/harbor/clawbench/test_control.py | 24 +++++++++++++ benchmarks/harbor/clawbench/verify-control.py | 35 ++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index 4f1bf82..c2f2eb5 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -156,6 +156,30 @@ def test_accepts_successful_calls_on_precreated_session(self) -> None: ): self.assertTrue(result[key], key) + def test_accepts_missing_terminal_observation_after_interception(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].append( + { + "tool_call_id": "computer-final", + "function_name": "mcp__kernel__computer_action", + "arguments": { + "session_id": "session-123", + "actions": [{"type": "click_mouse", "x": 10, "y": 10}], + }, + } + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + allowed_missing_observation_ids={"computer-final"}, + ) + self.assertTrue(result["observations_valid"]) + self.assertEqual( + result["expected_interrupted_observations"], ["computer-final"] + ) + self.assertEqual(result["unexpected_missing_observations"], []) + def test_rejects_another_session(self) -> None: result = verify.validate_control( self.trajectory("session-other"), diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index 395133d..db476a5 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -102,6 +102,7 @@ def validate_control( *, expected_session_id: str, expected_project_id: str, + allowed_missing_observation_ids: set[str] | None = None, ) -> dict[str, Any]: trajectory = trajectory or {} calls = _calls(trajectory) @@ -154,10 +155,14 @@ def validate_control( elif call.get("function_name") in BROWSER_TOOLS: successful_browser_calls += 1 + allowed_missing = allowed_missing_observation_ids or set() + unexpected_missing_observations = [ + call_id for call_id in missing_observations if call_id not in allowed_missing + ] observations_valid = ( successful_context_calls > 0 and successful_browser_calls > 0 - and not (missing_observations or duplicate_observations) + and not (unexpected_missing_observations or duplicate_observations) ) direct_http_patterns = re.compile( r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", @@ -180,6 +185,10 @@ def validate_control( "no_forbidden_kernel_tools": not forbidden_calls, "no_direct_http_automation": not direct_http_calls, "missing_observations": missing_observations, + "expected_interrupted_observations": [ + call_id for call_id in missing_observations if call_id in allowed_missing + ], + "unexpected_missing_observations": unexpected_missing_observations, "duplicate_observations": duplicate_observations, "error_observations": error_observations, "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], @@ -201,15 +210,33 @@ def main() -> int: lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") + interception = read_json(Path("/data/interception.json")) + agent_stop = read_json(Path("/data/agent-stop.json")) reward_path = VERIFIER_DIR / "reward.json" reward_metrics = read_json(reward_path) or {} session_id = str((browser or {}).get("session_id") or "") expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + all_calls = _calls(trajectory or {}) + browser_calls = [call for call in all_calls if call.get("function_name") in BROWSER_TOOLS] + terminal_call_id = browser_calls[-1].get("tool_call_id") if browser_calls else None + stop_detected_at = (agent_stop or {}).get("stop_detected_at") + intercepted_at = (interception or {}).get("intercepted_at") + stopped_after_interception = bool( + isinstance(stop_detected_at, (int, float)) + and isinstance(intercepted_at, (int, float)) + and 0 <= stop_detected_at - intercepted_at <= 5 + ) + allowed_missing = ( + {terminal_call_id} + if stopped_after_interception and isinstance(terminal_call_id, str) + else set() + ) atif = validate_control( trajectory, expected_session_id=session_id, expected_project_id=expected_project_id, + allowed_missing_observation_ids=allowed_missing, ) checks = { "kernel_mcp_context": atif["context_called"], @@ -242,6 +269,7 @@ def main() -> int: reward_metrics.get("intercepted") == 1 or (clawbench_result or {}).get("intercepted") is True ), + "agent_stopped_after_interception": stopped_after_interception, } infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") checks["infra_ok"] = infra_ok @@ -256,6 +284,11 @@ def main() -> int: "run_manifest": manifest, "browser_lifecycle": lifecycle, "clawbench_result": clawbench_result, + "interception": interception, + "agent_stop": agent_stop, + "stop_latency_seconds": ( + stop_detected_at - intercepted_at if stopped_after_interception else None + ), } (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) return 0 From 7475b8ab442fa7480864af283fda889f96a08ffe Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:24:42 +0000 Subject: [PATCH 17/22] Pin stop timing fix --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 6c442b3..832d2c5 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `4f39b26`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `6cf9dc5`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 5427ac6..faa6a2e 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-4f39b269abaab26cb886b643c0cfe6dde1b78698} +clawbench_ref=${CLAWBENCH_REF:-6cf9dc5c4d5b0ee9ae7d17bb8984691cdfad1796} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From 2bb487361cd5c255c89e75627bd82dd43c57f7b5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:49:35 +0000 Subject: [PATCH 18/22] Keep infrastructure health task-independent --- benchmarks/harbor/clawbench/verify-control.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index db476a5..b5db624 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -271,7 +271,11 @@ def main() -> int: ), "agent_stopped_after_interception": stopped_after_interception, } - infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") + infra_ok = all( + value + for name, value in checks.items() + if name not in {"clawbench_intercepted", "agent_stopped_after_interception"} + ) checks["infra_ok"] = infra_ok reward_metrics.update({name: float(value) for name, value in checks.items()}) From 37273e3f1829ef97a0b9dd1cbadcf0a57536891d Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:22:29 +0000 Subject: [PATCH 19/22] Encourage useful Playwright state returns --- src/lib/mcp/tools/playwright.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mcp/tools/playwright.ts b/src/lib/mcp/tools/playwright.ts index 585bff5..3fd0120 100644 --- a/src/lib/mcp/tools/playwright.ts +++ b/src/lib/mcp/tools/playwright.ts @@ -31,7 +31,7 @@ export function registerPlaywrightTool( code: z .string() .describe( - "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Example: `await page.goto('https://example.com'); return await page.title();` Return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page.", + "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Every invocation should return useful page state. After navigation or interaction, return a condensed accessibility snapshot of the relevant region, e.g. `await page.goto('https://example.com'); return await page.locator('main').ariaSnapshot();` or `await page.getByRole('button', { name: 'Submit' }).click(); return await page.locator('main').ariaSnapshot();`. For targeted reads, return a compact value or object. Do not dump the full DOM or body text.", ), session_id: z .string() From ac21a69082a11317946eee038d0746e7e14600ef Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:10:24 +0000 Subject: [PATCH 20/22] Strip smoke-task scaffolding, keep only the ClawBench Kernel MCP arm The smoke task duplicated what the ClawBench arm already proves. Drop its task definition, runner, verifier, fixtures, and MCP config, and drop stale ignore entries nothing writes. Document only the ClawBench flow. --- .gitignore | 2 - benchmarks/harbor/README.md | 49 +--- .../trajectory-codex-observation.json | 37 --- .../trajectory-error-observation.json | 37 --- .../trajectory-missing-observation.json | 33 --- .../bin/fixtures/trajectory-positive.json | 37 --- benchmarks/harbor/bin/test_verify_smoke.py | 59 ----- benchmarks/harbor/bin/verify-smoke.py | 227 ------------------ benchmarks/harbor/image/Dockerfile | 1 - benchmarks/harbor/mcp/kernel.json | 7 - benchmarks/harbor/prepare-task.py | 40 --- benchmarks/harbor/run-smoke.sh | 101 -------- benchmarks/harbor/smoke/environment/.gitkeep | 0 .../harbor/smoke/steps/run/instruction.md | 17 -- .../harbor/smoke/steps/run/tests/test.sh | 4 - .../harbor/smoke/steps/run/workdir/setup.sh | 12 - benchmarks/harbor/smoke/task.toml | 52 ---- 17 files changed, 11 insertions(+), 704 deletions(-) delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-error-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-positive.json delete mode 100644 benchmarks/harbor/bin/test_verify_smoke.py delete mode 100755 benchmarks/harbor/bin/verify-smoke.py delete mode 100644 benchmarks/harbor/mcp/kernel.json delete mode 100755 benchmarks/harbor/prepare-task.py delete mode 100755 benchmarks/harbor/run-smoke.sh delete mode 100644 benchmarks/harbor/smoke/environment/.gitkeep delete mode 100644 benchmarks/harbor/smoke/steps/run/instruction.md delete mode 100755 benchmarks/harbor/smoke/steps/run/tests/test.sh delete mode 100755 benchmarks/harbor/smoke/steps/run/workdir/setup.sh delete mode 100644 benchmarks/harbor/smoke/task.toml diff --git a/.gitignore b/.gitignore index 4e9fb8f..0240eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -111,8 +111,6 @@ mcp-key.pem # Harbor benchmark runtime data benchmarks/harbor/.image.env -benchmarks/harbor/.run.env -benchmarks/harbor/jobs/ benchmarks/harbor/image/source-sha # TypeScript incremental build cache diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 832d2c5..8646ea2 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -1,25 +1,18 @@ -# Harbor MCP benchmarks +# Harbor ClawBench benchmark -This directory runs stock Harbor agents against a locally built `kernel-mcp-server` in a single Hypeman sandbox. The smoke task makes two read-only calls through the configured stdio MCP server and writes standard Harbor job artifacts. +This directory runs stock Harbor agents (Claude Code, Codex) against a locally built `kernel-mcp-server` on a ClawBench task in a single Hypeman sandbox. The task starts from the Kernel-backed ClawBench Harbor adaptation, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. ## Requirements -- Harbor 0.21.0 -- `harbor-hypeman` 0.1.1 +- Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) - [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials +- A ClawBench checkout containing commit `6cf9dc5` (`kernel/ClawBench` PR #1) - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` +- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials - `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex -`run-smoke.sh` launches the pinned Harbor packages through `uvx`. To install the same versions as a persistent tool instead: - -```bash -uv tool install 'harbor==0.21.0' --with 'harbor-hypeman==0.1.1' -``` - -Set `HARBOR_BIN` only when intentionally testing a different Harbor installation. - ## Build the image ```bash @@ -28,12 +21,12 @@ Set `HARBOR_BIN` only when intentionally testing a different Harbor installation The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. -## Run the smoke task +## Run the ClawBench Kernel MCP arm ```bash -export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id -./benchmarks/harbor/run-smoke.sh claude-code -./benchmarks/harbor/run-smoke.sh codex +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run-control.sh claude-code \ + v2-1134-chapter-finder-redcross ``` Defaults: @@ -43,29 +36,9 @@ Defaults: | Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-terra` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`, or test a specific Claude Code release with `CLAUDE_BENCHMARK_VERSION`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. - -The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: - -- `steps/run/agent/trajectory.json` in ATIF format -- native agent logs and session data -- server stdout and stderr -- source SHA and Hypeman identity in `run-manifest.json` -- numeric Harbor rewards plus detailed `smoke-result.json` - -The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. - -## Run the ClawBench Kernel MCP arm - -The ClawBench arm starts from the Kernel-backed Harbor task produced by `clawbench-harbor-adapt`, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. - -```bash -export CLAWBENCH_REPO=../ClawBench -./benchmarks/harbor/clawbench/run-control.sh claude-code \ - v2-1134-chapter-finder-redcross -``` +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. -The ClawBench checkout must contain commit `6cf9dc5`, from `kernel/ClawBench` PR #1. The generated task: +`run-control.sh` adapts one ClawBench task with `clawbench-harbor-adapt`, converts it with `clawbench/prepare-control.py`, and runs it under Harbor. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json deleted file mode 100644 index d280135..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "[{'type': 'text', 'text': '{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}'}]" - }, - { - "source_call_id": "browsers-1", - "content": "[{'type': 'text', 'text': '{\"items\":[],\"has_more\":false}'}]" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json deleted file mode 100644 index 99f41ef..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" - }, - { - "source_call_id": "browsers-1", - "content": "{\"isError\":true,\"content\":[{\"type\":\"text\",\"text\":\"request failed\"}]}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json deleted file mode 100644 index c506f2e..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-positive.json b/benchmarks/harbor/bin/fixtures/trajectory-positive.json deleted file mode 100644 index a00aa56..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-positive.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"type\":\"text\",\"text\":\"{\\\"connection_scope\\\":{\\\"kind\\\":\\\"project\\\",\\\"project_id\\\":\\\"project-123\\\"}}\"}" - }, - { - "source_call_id": "browsers-1", - "content": "{\"type\":\"text\",\"text\":\"{\\\"items\\\":[],\\\"has_more\\\":false}\"}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py deleted file mode 100644 index 707468b..0000000 --- a/benchmarks/harbor/bin/test_verify_smoke.py +++ /dev/null @@ -1,59 +0,0 @@ -import importlib.util -import json -from pathlib import Path -import unittest - - -MODULE_PATH = Path(__file__).with_name("verify-smoke.py") -SPEC = importlib.util.spec_from_file_location("verify_smoke", MODULE_PATH) -assert SPEC and SPEC.loader -verify_smoke = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(verify_smoke) - -FIXTURES = Path(__file__).with_name("fixtures") - - -def load_fixture(name: str) -> dict: - return json.loads((FIXTURES / name).read_text()) - - -class VerifySmokeTest(unittest.TestCase): - def test_accepts_native_calls_with_paired_observations(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-positive.json"), "project-123" - ) - - self.assertTrue(proof["native_calls_present"]) - self.assertTrue(proof["observations_valid"]) - self.assertTrue(proof["context_scope_valid"]) - self.assertTrue(proof["manage_browsers_arguments_valid"]) - - def test_accepts_codex_serialized_observations(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-codex-observation.json"), "project-123" - ) - - self.assertTrue(proof["native_calls_present"]) - self.assertTrue(proof["observations_valid"]) - self.assertTrue(proof["context_scope_valid"]) - self.assertTrue(proof["manage_browsers_arguments_valid"]) - - def test_rejects_missing_observation(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-missing-observation.json"), "project-123" - ) - - self.assertFalse(proof["observations_valid"]) - self.assertEqual(proof["missing_observations"], ["browsers-1"]) - - def test_rejects_error_observation(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-error-observation.json"), "project-123" - ) - - self.assertFalse(proof["observations_valid"]) - self.assertEqual(proof["error_observations"], ["browsers-1"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py deleted file mode 100755 index a3a9e85..0000000 --- a/benchmarks/harbor/bin/verify-smoke.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import ast -import json -import os -from pathlib import Path -from typing import Any - -LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) -VERIFIER_DIR = LOGS_DIR / "verifier" -REQUIRED_TOOLS = { - "mcp__kernel__get_connection_context", - "mcp__kernel__manage_browsers", -} - - -def read_json(path: Path) -> dict[str, Any] | None: - try: - value = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return None - return value if isinstance(value, dict) else None - - -def _decode_tool_result_content(content: Any) -> Any: - value = content - for _ in range(6): - if isinstance(value, str): - try: - value = json.loads(value) - except json.JSONDecodeError: - try: - value = ast.literal_eval(value) - except (SyntaxError, ValueError): - return value - continue - if isinstance(value, dict) and value.get("type") == "text": - value = value.get("text") - continue - if ( - isinstance(value, list) - and len(value) == 1 - and isinstance(value[0], dict) - and value[0].get("type") == "text" - ): - value = value[0].get("text") - continue - return value - return value - - -def _contains_error(value: Any) -> bool: - if isinstance(value, dict): - if value.get("is_error") is True or value.get("isError") is True: - return True - if "error" in value and value["error"] not in (None, False, ""): - return True - return any(_contains_error(item) for item in value.values()) - if isinstance(value, list): - return any(_contains_error(item) for item in value) - if isinstance(value, str): - text = value.lstrip().lower() - return text.startswith(("[error]", "error:", "error in ")) - return False - - -def _observation_result_map(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - results: dict[str, list[dict[str, Any]]] = {} - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - observation = step.get("observation") - if not isinstance(observation, dict): - continue - for result in observation.get("results") or []: - if not isinstance(result, dict): - continue - source_call_id = result.get("source_call_id") - if isinstance(source_call_id, str): - results.setdefault(source_call_id, []).append(result) - return results - - -def _native_tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: - calls = [] - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - for call in step.get("tool_calls") or []: - if not isinstance(call, dict): - continue - if call.get("function_name") in REQUIRED_TOOLS: - calls.append(call) - return calls - - -def _manage_browsers_arguments_valid(call: dict[str, Any]) -> bool: - arguments = call.get("arguments") - return ( - isinstance(arguments, dict) - and arguments.get("action") == "list" - and arguments.get("status") == "active" - and arguments.get("limit") == 1 - ) - - -def validate_trajectory( - trajectory: dict[str, Any] | None, expected_project_id: str -) -> dict[str, Any]: - calls = _native_tool_calls(trajectory or {}) - calls_by_name = { - name: [call for call in calls if call.get("function_name") == name] - for name in REQUIRED_TOOLS - } - result_map = _observation_result_map(trajectory or {}) - missing_observations = [] - duplicate_observations = [] - error_observations = [] - context_scope_valid = True - browser_arguments_valid = True - - for call in calls: - call_id = call.get("tool_call_id") - results = result_map.get(call_id, []) if isinstance(call_id, str) else [] - if len(results) == 0: - missing_observations.append(call_id) - continue - if len(results) != 1: - duplicate_observations.append(call_id) - continue - result = results[0] - decoded = _decode_tool_result_content(result.get("content")) - if _contains_error(decoded) or _contains_error(result): - error_observations.append(call_id) - continue - if call.get("function_name") == "mcp__kernel__get_connection_context": - scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None - context_scope_valid = context_scope_valid and ( - bool(expected_project_id) - and isinstance(scope, dict) - and scope.get("kind") == "project" - and scope.get("project_id") == expected_project_id - ) - elif call.get("function_name") == "mcp__kernel__manage_browsers": - browser_arguments_valid = ( - browser_arguments_valid and _manage_browsers_arguments_valid(call) - ) - - native_calls_present = all(calls_by_name[name] for name in REQUIRED_TOOLS) - observations_valid = bool(calls) and not ( - missing_observations or duplicate_observations or error_observations - ) - return { - "native_calls_present": native_calls_present, - "observations_valid": observations_valid, - "context_scope_valid": context_scope_valid - and bool(calls_by_name["mcp__kernel__get_connection_context"]), - "manage_browsers_arguments_valid": browser_arguments_valid - and bool(calls_by_name["mcp__kernel__manage_browsers"]), - "missing_observations": missing_observations, - "duplicate_observations": duplicate_observations, - "error_observations": error_observations, - "tool_calls": [ - { - "tool_call_id": call.get("tool_call_id"), - "name": call.get("function_name"), - "arguments": call.get("arguments"), - } - for call in calls - ], - } - - -def main() -> int: - VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - report = read_json(LOGS_DIR / "artifacts/agent-report.json") - trajectory = read_json(LOGS_DIR / "agent/trajectory.json") - manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") - expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") - atif = validate_trajectory(trajectory, expected_project_id) - source_sha_matches = bool( - manifest - and manifest.get("kernel_mcp_server_sha") - == os.environ.get("KERNEL_MCP_SOURCE_SHA") - ) - hypeman_identity_present = bool( - manifest and manifest.get("hypeman_instance_name") - ) - - checks = { - "native_mcp_calls": atif["native_calls_present"], - "tool_observations": atif["observations_valid"], - "context_scope": atif["context_scope_valid"], - "manage_browsers_arguments": atif["manage_browsers_arguments_valid"], - "source_sha": source_sha_matches, - "hypeman_identity": hypeman_identity_present, - "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), - "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), - } - reward = 1.0 if all(checks.values()) else 0.0 - result = { - "reward": reward, - "checks": checks, - "atif": atif, - "agent_report": report, - "trajectory": { - "present": trajectory is not None, - "schema_version": (trajectory or {}).get("schema_version"), - "agent": (trajectory or {}).get("agent"), - }, - "run_manifest": manifest, - } - - (VERIFIER_DIR / "reward.txt").write_text(str(reward)) - (VERIFIER_DIR / "reward.json").write_text( - json.dumps( - {"reward": reward, **{name: float(value) for name, value in checks.items()}}, - indent=2, - ) - ) - (VERIFIER_DIR / "smoke-result.json").write_text(json.dumps(result, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index dfc0d5b..9bbc1c8 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -28,7 +28,6 @@ RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ bun run build \ && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ - && install -m 0755 benchmarks/harbor/bin/verify-smoke.py /usr/local/bin/verify-kernel-mcp-smoke \ && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA ENV NEXT_TELEMETRY_DISABLED=1 diff --git a/benchmarks/harbor/mcp/kernel.json b/benchmarks/harbor/mcp/kernel.json deleted file mode 100644 index 0ce2a9c..0000000 --- a/benchmarks/harbor/mcp/kernel.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "mcpServers": { - "kernel": { - "command": "/app/kernel-mcp-local" - } - } -} diff --git a/benchmarks/harbor/prepare-task.py b/benchmarks/harbor/prepare-task.py deleted file mode 100755 index 6ad8107..0000000 --- a/benchmarks/harbor/prepare-task.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import os -import shutil -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("output", type=Path) - args = parser.parse_args() - - source = Path(__file__).parent / "smoke" - output = args.output.resolve() - image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] - source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] - project_id = os.environ["KERNEL_MCP_BENCHMARK_PROJECT_ID"] - - if output.exists(): - shutil.rmtree(output) - shutil.copytree(source, output) - - config_path = output / "task.toml" - config = config_path.read_text() - config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) - config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) - config = config.replace("${KERNEL_MCP_BENCHMARK_PROJECT_ID}", project_id) - config_path.write_text(config) - - wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" - runtime_wrapper = output / "steps" / "run" / "workdir" / "kernel-mcp-local" - shutil.copy2(wrapper, runtime_wrapper) - runtime_wrapper.chmod(0o755) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh deleted file mode 100755 index c011da6..0000000 --- a/benchmarks/harbor/run-smoke.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -set -euo pipefail - -usage() { - echo "usage: $0 [job-name] [jobs-dir]" >&2 - exit 2 -} - -agent=${1:-} -[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage - -repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -benchmark_dir="$repo_root/benchmarks/harbor" -image_env="$benchmark_dir/.image.env" -[[ -f "$image_env" ]] || { - echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 - exit 1 -} - -set -a -source "$image_env" -set +a - -: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" -: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" - -case "$agent" in - claude-code) - if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then - echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 - exit 1 - fi - if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then - ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN - CLAUDE_FORCE_OAUTH=1 - export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH - fi - model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} - version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} - ;; - codex) - : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} - version=0.120.0 - ;; -esac - -if [[ -n "${HARBOR_BIN:-}" ]]; then - harbor_command=("$HARBOR_BIN") -elif command -v uvx >/dev/null 2>&1; then - harbor_command=( - uvx - --from "harbor==0.21.0" - --with "harbor-hypeman==0.1.1" - harbor - ) -else - echo "uvx not found; install uv or set HARBOR_BIN" >&2 - exit 1 -fi - -job_name=${2:-${agent}-smoke-$(date -u +%Y%m%dT%H%M%SZ)} -jobs_dir=${3:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-harbor-jobs}} -runtime_task=$(mktemp -d) -runtime_env=$(mktemp) -trap 'rm -rf "$runtime_task"; rm -f "$runtime_env"' EXIT - -export KERNEL_MCP_BENCHMARK_IMAGE KERNEL_MCP_SOURCE_SHA -python3 "$benchmark_dir/prepare-task.py" "$runtime_task" - -cat >"$runtime_env" <"$key_dir/api-key" -chmod 0600 "$key_dir/api-key" - -/usr/local/bin/start-kernel-mcp-server -printf 'ready\n' >/logs/kernel-mcp/ready -rm -f /app/setup.sh diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml deleted file mode 100644 index 055bd16..0000000 --- a/benchmarks/harbor/smoke/task.toml +++ /dev/null @@ -1,52 +0,0 @@ -schema_version = "1.4" -source = "kernel-mcp-benchmarks" -artifacts = ["/logs/kernel-mcp"] -multi_step_reward_strategy = "final" - -[task] -name = "kernel-mcp/local-connection-smoke" -description = "Verify an agent can call a locally running Kernel MCP server" -keywords = ["kernel", "mcp", "harbor", "smoke"] - -[metadata] -benchmark = "kernel-mcp-local-connection" -kernel_mcp_server_sha = "${KERNEL_MCP_SOURCE_SHA}" - -[environment] -docker_image = "${KERNEL_MCP_BENCHMARK_IMAGE}" -network_mode = "public" -workdir = "/app" -build_timeout_sec = 1200.0 -cpus = 2 -memory_mb = 4096 -storage_mb = 8192 - -[environment.env] -KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" -KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" -KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" -KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" -KERNEL_PROJECT = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" -API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" -REDIS_URL = "redis://127.0.0.1:6379" -CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY = "pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ" -MANAGED_AUTH_APP_ORIGIN = "http://127.0.0.1:3002" -NEXT_TELEMETRY_DISABLED = "1" - -[[steps]] -name = "run" - -[steps.agent] -timeout_sec = 300.0 - -[steps.verifier] -timeout_sec = 60.0 - -[steps.healthcheck] -command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp" -interval_sec = 2.0 -timeout_sec = 5.0 -start_period_sec = 1.0 -start_interval_sec = 1.0 -retries = 10 From 9e91f526e620dd3a42ba3a965721f6a139451e12 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:56:21 +0000 Subject: [PATCH 21/22] Make ClawBench benchmark DOM-only --- benchmarks/harbor/README.md | 16 +++++--- .../harbor/clawbench/prepare-control.py | 8 ++-- benchmarks/harbor/clawbench/run-control.sh | 38 +++++++++++-------- benchmarks/harbor/clawbench/test_control.py | 21 +++++----- benchmarks/harbor/clawbench/verify-control.py | 8 ++-- 5 files changed, 52 insertions(+), 39 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 8646ea2..1453451 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -6,7 +6,7 @@ This directory runs stock Harbor agents (Claude Code, Codex) against a locally b - Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) - [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials -- A ClawBench checkout containing commit `6cf9dc5` (`kernel/ClawBench` PR #1) +- A ClawBench checkout containing commit `df6743f` (`kernel/ClawBench` PR #1) - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials @@ -34,14 +34,20 @@ Defaults: | Agent | Version | Model | | ----------- | ------: | ----------------- | | Claude Code | 2.1.238 | `claude-sonnet-5` | -| Codex | 0.120.0 | `gpt-5.6-terra` | +| Codex | 0.120.0 | `gpt-5.6-luna` | Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. -`run-control.sh` adapts one ClawBench task with `clawbench-harbor-adapt`, converts it with `clawbench/prepare-control.py`, and runs it under Harbor. The generated task: +Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: -- exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` -- disables browser lifecycle and managed-auth toolsets +```bash +HARBOR_N_CONCURRENT=10 ./benchmarks/harbor/clawbench/run-control.sh codex all +``` + +`run-control.sh` adapts the selected ClawBench tasks with `clawbench-harbor-adapt`, converts them with `clawbench/prepare-control.py`, and runs them under Harbor. Each generated task: + +- exposes `get_connection_context` and `execute_playwright_code` +- disables coordinate-based computer actions, browser lifecycle, and managed-auth toolsets - instructs the agent to read `./my-info/kernel_browser.json` and use that session ID - instructs account tasks to use the supplied PurelyMail credentials instead of managed auth - verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 040a9d5..26fd412 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -6,7 +6,7 @@ import shutil from pathlib import Path -ENABLED_TOOLSETS = "playwright computer" +ENABLED_TOOLSETS = "playwright" def _drop_mcp_servers(task_toml: str) -> str: @@ -119,9 +119,9 @@ def _patch_instruction(instruction: str) -> str: Kernel MCP benchmark arm: - Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. - Call `get_connection_context` once before taking any browser action. -- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. -- Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. -- Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. +- Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. - Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. - Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. - Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index faa6a2e..3dbd888 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -2,7 +2,7 @@ set -euo pipefail usage() { - echo "usage: $0 [task-id] [job-name] [jobs-dir]" >&2 + echo "usage: $0 [task-id|all] [job-name] [jobs-dir]" >&2 exit 2 } @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-6cf9dc5c4d5b0ee9ae7d17bb8984691cdfad1796} +clawbench_ref=${CLAWBENCH_REF:-df6743fd8abcd09cb7636ef8c310dd4db016162c} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 @@ -59,7 +59,7 @@ case "$agent" in ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-luna} version=${CODEX_BENCHMARK_VERSION:-0.120.0} ;; esac @@ -69,23 +69,29 @@ runtime_env=$(mktemp) trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT dataset="$runtime_root/dataset" -uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ - --output-dir "$dataset" \ - --task-ids "$task_id" \ - --browser-runtime kernel \ - --browser-runtime-options '{"stealth": true}' \ +adapt_args=( + --output-dir "$dataset" + --browser-runtime kernel + --browser-runtime-options '{"stealth": true}' --overwrite +) +if [[ "$task_id" != "all" ]]; then + adapt_args+=(--task-ids "$task_id") +fi +uv --directory "$clawbench_repo" run clawbench-harbor-adapt "${adapt_args[@]}" -task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) -[[ -n "$task_dir" ]] || { - echo "ClawBench did not generate task $task_id" >&2 +mapfile -t task_dirs < <(find "$dataset" -mindepth 1 -maxdepth 1 -type d | sort) +((${#task_dirs[@]} > 0)) || { + echo "ClawBench did not generate tasks for $task_id" >&2 exit 1 } -python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ - --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ - --server-sha "$KERNEL_MCP_SOURCE_SHA" \ - --clawbench-sha "$clawbench_ref" +for task_dir in "${task_dirs[@]}"; do + python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" +done export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} @@ -126,7 +132,7 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-40m}" \ --env-file "$runtime_env" \ --job-name "$job_name" \ --jobs-dir "$jobs_dir" \ - --n-concurrent 1 \ + --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \ --max-retries 0 \ --delete \ --yes diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index c2f2eb5..bb6941b 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -73,9 +73,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: self.assertIn('name = "kernel"', task_toml) self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) self.assertNotIn("@playwright/mcp", task_toml) - self.assertIn( - 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml - ) + self.assertIn('KERNEL_MCP_ENABLED_TOOLSETS = "playwright"', task_toml) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) @@ -90,7 +88,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: instruction = (step / "instruction.md").read_text() self.assertIn("WaitForMcpServers", instruction) self.assertIn("existing `session_id`", instruction) - self.assertIn("Use only Kernel MCP browser-control tools", instruction) + self.assertIn("Use Kernel MCP `execute_playwright_code`", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) self.assertIn("Do not call `fetch`", instruction) @@ -160,11 +158,11 @@ def test_accepts_missing_terminal_observation_after_interception(self) -> None: trajectory = self.trajectory() trajectory["steps"][0]["tool_calls"].append( { - "tool_call_id": "computer-final", - "function_name": "mcp__kernel__computer_action", + "tool_call_id": "playwright-final", + "function_name": "mcp__kernel__execute_playwright_code", "arguments": { "session_id": "session-123", - "actions": [{"type": "click_mouse", "x": 10, "y": 10}], + "code": "await page.getByRole('button').click()", }, } ) @@ -172,11 +170,11 @@ def test_accepts_missing_terminal_observation_after_interception(self) -> None: trajectory, expected_session_id="session-123", expected_project_id="project-123", - allowed_missing_observation_ids={"computer-final"}, + allowed_missing_observation_ids={"playwright-final"}, ) self.assertTrue(result["observations_valid"]) self.assertEqual( - result["expected_interrupted_observations"], ["computer-final"] + result["expected_interrupted_observations"], ["playwright-final"] ) self.assertEqual(result["unexpected_missing_observations"], []) @@ -214,6 +212,11 @@ def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: "function_name": "mcp__kernel__manage_browsers", "arguments": {"action": "list"}, }, + { + "tool_call_id": "computer-action", + "function_name": "mcp__kernel__computer_action", + "arguments": {"session_id": "session-123", "actions": []}, + }, ] ) result = verify.validate_control( diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index b5db624..7660925 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -11,11 +11,9 @@ LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) VERIFIER_DIR = LOGS_DIR / "verifier" CONTEXT_TOOL = "mcp__kernel__get_connection_context" -BROWSER_TOOLS = { - "mcp__kernel__execute_playwright_code", - "mcp__kernel__computer_action", -} +BROWSER_TOOLS = {"mcp__kernel__execute_playwright_code"} FORBIDDEN_KERNEL_TOOLS = { + "mcp__kernel__computer_action", "mcp__kernel__manage_browsers", "mcp__kernel__manage_auth_connections", "mcp__kernel__open_auth_login", @@ -257,7 +255,7 @@ def main() -> int: "kernel_mcp_toolset_allowlist": bool( manifest and set(str(manifest.get("enabled_toolsets", "")).split()) - == {"playwright", "computer"} + == {"playwright"} ), "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), "browser_deleted": bool( From b5799f9b7a66e02a33b627a02ffaaa9615db6921 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:01:15 +0000 Subject: [PATCH 22/22] Allow full ClawBench suite to finish --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 1453451..a30fed5 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -36,7 +36,7 @@ Defaults: | Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-luna` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Single-task runs have a 40-minute wall-clock limit; full-suite runs default to 6 hours. Change either with `HARBOR_BENCHMARK_TIMEOUT`. Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 3dbd888..0406273 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -122,7 +122,13 @@ job_name=${3:-kernel-mcp-${agent}-${task_id}-$(date -u +%Y%m%dT%H%M%SZ)} jobs_dir=${4:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-clawbench-jobs}} mkdir -p "$jobs_dir" -timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-40m}" \ +if [[ "$task_id" == "all" ]]; then + default_timeout=6h +else + default_timeout=40m +fi + +timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_timeout}" \ uvx --from "harbor==0.21.0" --with "harbor-hypeman==0.1.1" harbor run \ --path "$dataset" \ --agent "$agent" \