diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b58cdc1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.github +.next +node_modules +coverage +.env* +*.log +benchmarks/harbor/.image.env diff --git a/.gitignore b/.gitignore index 4069f2b..0240eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Dependencies node_modules/ +__pycache__/ +*.py[cod] npm-debug.log* yarn-debug.log* yarn-error.log* @@ -107,5 +109,9 @@ Makefile # private key mcp-key.pem +# Harbor benchmark runtime data +benchmarks/harbor/.image.env +benchmarks/harbor/image/source-sha + # TypeScript incremental build cache tsconfig.tsbuildinfo diff --git a/README.md b/README.md index f9bd672..06a6420 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ Each Kernel feature has a single `manage_*` tool with an `action` parameter, kee One additional Managed Auth helper (`begin_auth_login`) is marked app-only (`_meta.ui.visibility: ["app"]`); it refuses to execute on hosts that do not declare MCP Apps support. The App forwards the server-issued signed flow checkpoint to the shared `manage_auth_connections` `wait` action, so flow identity and terminal-state decisions stay on the server. -Self-hosted deployments can hide sensitive tool families by setting `KERNEL_MCP_DISABLED_TOOLSETS` to a comma-separated list. For example, `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` prevents `manage_api_keys` from being registered. +Self-hosted deployments can select tool families with `KERNEL_MCP_ENABLED_TOOLSETS` or hide them with `KERNEL_MCP_DISABLED_TOOLSETS`. Both accept comma- or space-separated toolset names and standalone aliases. For example, `KERNEL_MCP_ENABLED_TOOLSETS="playwright computer"` exposes browser-control tools without browser lifecycle or managed-auth tools, while `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` only removes `manage_api_keys`. `get_connection_context` remains available in either mode. Call `get_connection_context` before deciding whether to create or select a project. Its canonical `connection_scope` reports whether the connection is organization-wide or fixed to a project. Project-scoped tools advertise an optional `project` (name or ID) and a deprecated `project_id`: organization-wide connections may omit them to preserve organization-wide reads and API default-project behavior, while fixed-project connections may omit them or pass the matching project. Project resources use project-qualified `kernel://orgs/{organizationId}/projects/{projectId}/...` URIs. Authorization remains enforced by the Kernel API; selecting a project never grants access to it. diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md new file mode 100644 index 0000000..a30fed5 --- /dev/null +++ b/benchmarks/harbor/README.md @@ -0,0 +1,55 @@ +# Harbor ClawBench benchmark + +This directory runs stock Harbor agents (Claude Code, Codex) against a locally built `kernel-mcp-server` on a ClawBench task in a single Hypeman sandbox. The task starts from the Kernel-backed ClawBench Harbor adaptation, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. + +## Requirements + +- Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) +- [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials +- A ClawBench checkout containing commit `df6743f` (`kernel/ClawBench` PR #1) +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project +- `KERNEL_MCP_BENCHMARK_PROJECT_ID` +- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials +- `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code +- `OPENAI_API_KEY` for Codex + +## Build the image + +```bash +./benchmarks/harbor/build-image.sh +``` + +The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. + +## Run the ClawBench Kernel MCP arm + +```bash +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run-control.sh claude-code \ + v2-1134-chapter-finder-redcross +``` + +Defaults: + +| Agent | Version | Model | +| ----------- | ------: | ----------------- | +| Claude Code | 2.1.238 | `claude-sonnet-5` | +| Codex | 0.120.0 | `gpt-5.6-luna` | + +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Single-task runs have a 40-minute wall-clock limit; full-suite runs default to 6 hours. Change either with `HARBOR_BENCHMARK_TIMEOUT`. + +Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: + +```bash +HARBOR_N_CONCURRENT=10 ./benchmarks/harbor/clawbench/run-control.sh codex all +``` + +`run-control.sh` adapts the selected ClawBench tasks with `clawbench-harbor-adapt`, converts them with `clawbench/prepare-control.py`, and runs them under Harbor. Each generated task: + +- exposes `get_connection_context` and `execute_playwright_code` +- disables coordinate-based computer actions, browser lifecycle, and managed-auth toolsets +- instructs the agent to read `./my-info/kernel_browser.json` and use that session ID +- instructs account tasks to use the supplied PurelyMail credentials instead of managed auth +- verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion + +Outputs use the normal Harbor job directory and add `kernel-mcp-control-result.json`, Kernel MCP logs, source manifests, and same-session metrics to the ClawBench verifier artifacts. diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local new file mode 100755 index 0000000..9cb8c06 --- /dev/null +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +key_file=/run/kernel-mcp-benchmark/api-key +if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then + KERNEL_API_KEY=$(cat "$key_file") + export KERNEL_API_KEY +fi +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +exec mcp-remote \ + http://127.0.0.1:3002/mcp \ + --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server new file mode 100755 index 0000000..2096588 --- /dev/null +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +log_dir=/logs/kernel-mcp +key_dir=/run/kernel-mcp-benchmark +mkdir -p "$log_dir" /logs/artifacts "$key_dir" +chmod 0777 "$log_dir" /logs/artifacts +chmod 0700 "$key_dir" +printf '%s' "$KERNEL_API_KEY" >"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ + --logfile "$log_dir/redis.log" --dir /tmp + +export CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-sk_test_kernel_mcp_benchmark_local_only} +export NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ} + +cd /opt/kernel-mcp-server +nohup ./node_modules/.bin/next start -p 3002 \ + >"$log_dir/server.stdout.log" \ + 2>"$log_dir/server.stderr.log" & +echo $! >"$log_dir/server.pid" + +for _ in $(seq 1 90); do + if curl -fsS -X POST http://127.0.0.1:3002/mcp \ + -H "Authorization: Bearer ${KERNEL_API_KEY}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + --data '{"jsonrpc":"2.0","id":"benchmark-healthcheck","method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"harbor-healthcheck","version":"1.0.0"}}}' \ + >"$log_dir/initialize-response.txt"; then + break + fi + sleep 1 +done + +if [ ! -s "$log_dir/initialize-response.txt" ]; then + echo "Kernel MCP server did not become ready" >&2 + tail -100 "$log_dir/server.stderr.log" >&2 || true + exit 1 +fi + +python3 - <<'PY' +import json +import os +import platform +from datetime import datetime, timezone +from pathlib import Path + +browser_path = Path("/my-info/kernel_browser.json") +try: + browser = json.loads(browser_path.read_text()) +except (OSError, json.JSONDecodeError): + browser = {} + +manifest = { + "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "clawbench_source_sha": os.environ.get("CLAWBENCH_SOURCE_SHA", ""), + "browser_session_id": browser.get("session_id"), + "enabled_toolsets": os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", ""), + "disabled_toolsets": os.environ.get("KERNEL_MCP_DISABLED_TOOLSETS", ""), + "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), + "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), + "sandbox_hostname": platform.node(), + "started_at": datetime.now(timezone.utc).isoformat(), +} +Path("/logs/kernel-mcp/run-manifest.json").write_text(json.dumps(manifest, indent=2)) +PY + +printf 'ready\n' >"$log_dir/ready" diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh new file mode 100755 index 0000000..d8319af --- /dev/null +++ b/benchmarks/harbor/build-image.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" + +source_sha=$(git rev-parse HEAD) +source_sha_file=benchmarks/harbor/image/source-sha +build_log=$(mktemp) +trap 'rm -f "$source_sha_file" "$build_log"' EXIT + +printf '%s\n' "$source_sha" >"$source_sha_file" + +set +e +hypeman build \ + --file benchmarks/harbor/image/Dockerfile \ + --cpus 4 \ + --memory 8192 \ + --timeout 1800 \ + . 2>&1 | tee "$build_log" +build_status=${PIPESTATUS[0]} +set -e + +build_id=$(sed -n -E 's/^Build (ID|started): //p' "$build_log" | tail -1) +if [[ -z "$build_id" ]]; then + echo "Hypeman did not return a build ID" >&2 + exit 1 +fi + +image_ref="docker.io/builds/$build_id:latest" +if ((build_status != 0)); then + echo "Build record failed; checking for a delayed ready image for up to 5 minutes" >&2 + image_ready=false + for _ in $(seq 1 30); do + if hypeman --format json image list | python3 -c ' +import json +import sys + +image_ref = sys.argv[1] +expected = {image_ref, image_ref.removeprefix("docker.io/")} +images = json.load(sys.stdin) +raise SystemExit( + 0 + if any(image.get("name") in expected and image.get("status") == "ready" for image in images) + else 1 +) +' "$image_ref" + then + image_ready=true + break + fi + sleep 10 + done + if [[ "$image_ready" != true ]]; then + exit "$build_status" + fi +fi + +cat >benchmarks/harbor/.image.env < str: + lines = task_toml.splitlines() + output: list[str] = [] + dropping = False + for line in lines: + if line.strip() == "[[environment.mcp_servers]]": + dropping = True + continue + if dropping and line.startswith("["): + dropping = False + if not dropping: + output.append(line) + return "\n".join(output).rstrip() + "\n" + + +def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_sha: str) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + inserted_image = False + inserted_env = False + for line in lines: + output.append(line) + if line.strip() == "[environment]": + output.append(f"docker_image = {json.dumps(image)}") + inserted_image = True + elif line.strip() == "[environment.env]": + output.extend( + [ + f"KERNEL_MCP_BENCHMARK_IMAGE = {json.dumps(image)}", + f"KERNEL_MCP_SOURCE_SHA = {json.dumps(server_sha)}", + f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", + f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", + 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', + 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'REDIS_URL = "redis://127.0.0.1:6379"', + ] + ) + inserted_env = True + if not inserted_image or not inserted_env: + raise ValueError("generated task is missing Harbor environment sections") + output.extend( + [ + "", + "[[environment.mcp_servers]]", + 'name = "kernel"', + 'transport = "stdio"', + 'command = "/usr/local/bin/kernel-mcp-local"', + "args = []", + ] + ) + return "\n".join(output).rstrip() + "\n" + + +def _patch_setup(setup: str) -> str: + install = """install_clawbench_runtime() { + mkdir -p /app/src + rm -rf /app/src/runtime-server /app/src/chrome-extension /app/src/shared /app/src/harbor + cp -a /runtime-server /app/src/runtime-server + cp -a /chrome-extension /app/src/chrome-extension + cp -a /shared /app/src/shared + cp -a /harbor /app/src/harbor + chmod +x /app/src/harbor/*.sh /app/src/harbor/*.py + cd /app/src/runtime-server + UV_PYTHON_PREFERENCE=only-system uv sync --frozen + uv pip install --python .venv/bin/python fpdf2 + cd / +} + +install_clawbench_runtime +""" + marker = "mkdir -p /data /logs/verifier /extra_info\n" + if marker not in setup: + raise ValueError("generated setup script is missing directory initialization") + setup = setup.replace(marker, marker + "\n" + install, 1) + runtime_marker = "/app/src/harbor/start-runtime.sh\n" + if runtime_marker not in setup: + raise ValueError("generated setup script is missing runtime startup") + return setup.replace( + runtime_marker, + runtime_marker + "\nstart-kernel-mcp-server\n", + 1, + ) + + +def _patch_verifier(test_script: str) -> str: + verify_marker = "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + if verify_marker not in test_script: + raise ValueError("generated verifier script is missing ClawBench verification") + return test_script.replace( + verify_marker, + verify_marker + + "mkdir -p /logs/verifier/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /logs/verifier/kernel-mcp/\n" + + "/app/src/runtime-server/.venv/bin/python " + + "/app/src/harbor/verify-kernel-mcp-control.py\n", + 1, + ) + + +def _patch_instruction(instruction: str) -> str: + instruction = instruction.replace( + "Use only Playwright MCP browser tools plus reading files", + "Use only Kernel MCP browser-control tools plus reading files", + ) + return instruction.rstrip() + """ + +--- +Kernel MCP benchmark arm: +- Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. +- Call `get_connection_context` once before taking any browser action. +- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. +- Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. +- Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. +- Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. +- Complete and submit the task through the existing browser, then stop. +""" + + +def transform_task(task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str) -> None: + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.unlink(missing_ok=True) + + task_toml_path = task_dir / "task.toml" + task_toml = _drop_mcp_servers(task_toml_path.read_text()) + task_toml_path.write_text( + _add_environment( + task_toml, + image=image, + server_sha=server_sha, + clawbench_sha=clawbench_sha, + ) + ) + + step_dir = task_dir / "steps" / "run" + setup_path = step_dir / "workdir" / "setup.sh" + setup_path.write_text(_patch_setup(setup_path.read_text())) + setup_path.chmod(0o755) + + test_path = step_dir / "tests" / "test.sh" + test_path.write_text(_patch_verifier(test_path.read_text())) + test_path.chmod(0o755) + + instruction_path = step_dir / "instruction.md" + instruction_path.write_text(_patch_instruction(instruction_path.read_text())) + + verifier_source = Path(__file__).with_name("verify-control.py") + verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-control.py" + shutil.copy2(verifier_source, verifier_target) + verifier_target.chmod(0o755) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert a Kernel-backed ClawBench Harbor task to the Kernel MCP arm") + parser.add_argument("task_dir", type=Path) + parser.add_argument("--image", required=True) + parser.add_argument("--server-sha", required=True) + parser.add_argument("--clawbench-sha", required=True) + args = parser.parse_args() + transform_task( + args.task_dir, + image=args.image, + server_sha=args.server_sha, + clawbench_sha=args.clawbench_sha, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh new file mode 100755 index 0000000..0406273 --- /dev/null +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -0,0 +1,144 @@ +#!/bin/bash +set -euo pipefail + +usage() { + echo "usage: $0 [task-id|all] [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage +task_id=${2:-v2-1134-chapter-finder-redcross} + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +clawbench_ref=${CLAWBENCH_REF:-df6743fd8abcd09cb7636ef8c310dd4db016162c} + +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} +[[ -d "$clawbench_repo/.git" || -f "$clawbench_repo/.git" ]] || { + echo "ClawBench checkout not found at $clawbench_repo" >&2 + exit 1 +} +git -C "$clawbench_repo" merge-base --is-ancestor "$clawbench_ref" HEAD || { + echo "ClawBench checkout must contain $clawbench_ref" >&2 + exit 1 +} + +if [[ -f "$clawbench_repo/.env" ]]; then + set -a + source "$clawbench_repo/.env" + set +a +fi +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" +: "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" +: "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" + +case "$agent" in + claude-code) + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-luna} + version=${CODEX_BENCHMARK_VERSION:-0.120.0} + ;; +esac + +runtime_root=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT + +dataset="$runtime_root/dataset" +adapt_args=( + --output-dir "$dataset" + --browser-runtime kernel + --browser-runtime-options '{"stealth": true}' + --overwrite +) +if [[ "$task_id" != "all" ]]; then + adapt_args+=(--task-ids "$task_id") +fi +uv --directory "$clawbench_repo" run clawbench-harbor-adapt "${adapt_args[@]}" + +mapfile -t task_dirs < <(find "$dataset" -mindepth 1 -maxdepth 1 -type d | sort) +((${#task_dirs[@]} > 0)) || { + echo "ClawBench did not generate tasks for $task_id" >&2 + exit 1 +} + +for task_dir in "${task_dirs[@]}"; do + python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" +done + +export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY +export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} +export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} +export KERNEL_MCP_BENCHMARK_PROJECT_ID + +cat >"$runtime_env" < ModuleType: + spec = importlib.util.spec_from_file_location(name, HERE / filename) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +prepare = load_module("prepare_control", "prepare-control.py") +verify = load_module("verify_control", "verify-control.py") + + +class PrepareControlTest(unittest.TestCase): + def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: + with tempfile.TemporaryDirectory() as temp: + task = Path(temp) + environment = task / "environment" + step = task / "steps" / "run" + (step / "workdir").mkdir(parents=True) + (step / "tests").mkdir() + (step / "instruction.md").write_text( + "Use only Playwright MCP browser tools plus reading files under ./my-info/.\n" + ) + (environment / "harbor").mkdir(parents=True) + (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") + (task / "task.toml").write_text( + """[environment] +workdir = "/" + +[environment.env] +KERNEL_API_KEY = "${KERNEL_API_KEY}" + +[[steps]] +name = "run" + +[[environment.mcp_servers]] +name = "playwright" +transport = "stdio" +command = "npx" +args = ["-y", "@playwright/mcp@0.0.79"] +""" + ) + (step / "workdir" / "setup.sh").write_text( + "#!/bin/bash\nmkdir -p /data /logs/verifier /extra_info\n" + "/app/src/harbor/start-runtime.sh\n" + ) + (step / "tests" / "test.sh").write_text( + "#!/bin/bash\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + ) + + prepare.transform_task( + task, + image="docker.io/builds/image:latest", + server_sha="server-sha", + clawbench_sha="clawbench-sha", + ) + + task_toml = (task / "task.toml").read_text() + self.assertFalse((environment / "Dockerfile").exists()) + self.assertIn('docker_image = "docker.io/builds/image:latest"', task_toml) + self.assertIn('name = "kernel"', task_toml) + self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) + self.assertNotIn("@playwright/mcp", task_toml) + self.assertIn('KERNEL_MCP_ENABLED_TOOLSETS = "playwright"', task_toml) + self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) + self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) + self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) + + setup = (step / "workdir" / "setup.sh").read_text() + self.assertIn("install_clawbench_runtime", setup) + self.assertIn("start-kernel-mcp-server", setup) + test_script = (step / "tests" / "test.sh").read_text() + self.assertIn("verify-kernel-mcp-control.py", test_script) + self.assertIn("/logs/verifier/kernel-mcp", test_script) + instruction = (step / "instruction.md").read_text() + self.assertIn("WaitForMcpServers", instruction) + self.assertIn("existing `session_id`", instruction) + self.assertIn("Use Kernel MCP `execute_playwright_code`", instruction) + self.assertIn("PurelyMail-backed credentials", instruction) + self.assertIn("Do not use Kernel managed auth", instruction) + self.assertIn("Do not call `fetch`", instruction) + self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) + + +class VerifyControlTest(unittest.TestCase): + def trajectory(self, session_id: str = "session-123") -> dict: + return { + "steps": [ + { + "tool_calls": [ + { + "tool_call_id": "context-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {}, + }, + { + "tool_call_id": "playwright-1", + "function_name": "mcp__kernel__execute_playwright_code", + "arguments": { + "session_id": session_id, + "code": "await page.goto('https://example.com')", + }, + }, + ], + "observation": { + "results": [ + { + "source_call_id": "context-1", + "content": { + "connection_scope": { + "kind": "project", + "project_id": "project-123", + } + }, + }, + { + "source_call_id": "playwright-1", + "content": [{"type": "text", "text": "{\"ok\": true}"}], + }, + ] + }, + } + ] + } + + def test_accepts_successful_calls_on_precreated_session(self) -> None: + result = verify.validate_control( + self.trajectory(), + expected_session_id="session-123", + expected_project_id="project-123", + ) + for key in ( + "context_called", + "browser_control_called", + "observations_valid", + "context_scope_valid", + "same_session", + "no_playwright_mcp", + "no_forbidden_kernel_tools", + "no_direct_http_automation", + ): + self.assertTrue(result[key], key) + + def test_accepts_missing_terminal_observation_after_interception(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].append( + { + "tool_call_id": "playwright-final", + "function_name": "mcp__kernel__execute_playwright_code", + "arguments": { + "session_id": "session-123", + "code": "await page.getByRole('button').click()", + }, + } + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + allowed_missing_observation_ids={"playwright-final"}, + ) + self.assertTrue(result["observations_valid"]) + self.assertEqual( + result["expected_interrupted_observations"], ["playwright-final"] + ) + self.assertEqual(result["unexpected_missing_observations"], []) + + def test_rejects_another_session(self) -> None: + result = verify.validate_control( + self.trajectory("session-other"), + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["same_session"]) + + def test_rejects_direct_http_inside_playwright_code(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"][1]["arguments"]["code"] = ( + "return await page.evaluate(() => fetch('/api'))" + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_direct_http_automation"]) + + def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].extend( + [ + { + "tool_call_id": "direct-playwright", + "function_name": "mcp__playwright__browser_navigate", + "arguments": {}, + }, + { + "tool_call_id": "browser-list", + "function_name": "mcp__kernel__manage_browsers", + "arguments": {"action": "list"}, + }, + { + "tool_call_id": "computer-action", + "function_name": "mcp__kernel__computer_action", + "arguments": {"session_id": "session-123", "actions": []}, + }, + ] + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_playwright_mcp"]) + self.assertFalse(result["no_forbidden_kernel_tools"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py new file mode 100755 index 0000000..7660925 --- /dev/null +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import ast +import json +import os +import re +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" +CONTEXT_TOOL = "mcp__kernel__get_connection_context" +BROWSER_TOOLS = {"mcp__kernel__execute_playwright_code"} +FORBIDDEN_KERNEL_TOOLS = { + "mcp__kernel__computer_action", + "mcp__kernel__manage_browsers", + "mcp__kernel__manage_auth_connections", + "mcp__kernel__open_auth_login", +} + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _decode_content(content: Any) -> Any: + value = content + for _ in range(6): + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + try: + value = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value + continue + if isinstance(value, dict) and value.get("type") == "text": + value = value.get("text") + continue + if ( + isinstance(value, list) + and len(value) == 1 + and isinstance(value[0], dict) + and value[0].get("type") == "text" + ): + value = value[0].get("text") + continue + return value + return value + + +def _contains_error(value: Any) -> bool: + if isinstance(value, dict): + if value.get("is_error") is True or value.get("isError") is True: + return True + if value.get("error") not in (None, False, ""): + return True + return any(_contains_error(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error(item) for item in value) + if isinstance(value, str): + return value.lstrip().lower().startswith(("[error]", "error:", "error in ")) + return False + + +def _calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + calls.extend(call for call in step.get("tool_calls") or [] if isinstance(call, dict)) + return calls + + +def _results(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + results: dict[str, list[dict[str, Any]]] = {} + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + observation = step.get("observation") + if not isinstance(observation, dict): + continue + for result in observation.get("results") or []: + if not isinstance(result, dict): + continue + call_id = result.get("source_call_id") + if isinstance(call_id, str): + results.setdefault(call_id, []).append(result) + return results + + +def validate_control( + trajectory: dict[str, Any] | None, + *, + expected_session_id: str, + expected_project_id: str, + allowed_missing_observation_ids: set[str] | None = None, +) -> dict[str, Any]: + trajectory = trajectory or {} + calls = _calls(trajectory) + result_map = _results(trajectory) + kernel_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__kernel__") + ] + context_calls = [call for call in kernel_calls if call.get("function_name") == CONTEXT_TOOL] + browser_calls = [call for call in kernel_calls if call.get("function_name") in BROWSER_TOOLS] + playwright_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__playwright__") + ] + forbidden_calls = [ + call for call in kernel_calls if call.get("function_name") in FORBIDDEN_KERNEL_TOOLS + ] + + missing_observations: list[Any] = [] + duplicate_observations: list[Any] = [] + error_observations: list[Any] = [] + context_scope_valid = bool(expected_project_id) + same_session = bool(expected_session_id and browser_calls) and all( + isinstance(call.get("arguments"), dict) + and call["arguments"].get("session_id") == expected_session_id + for call in browser_calls + ) + successful_context_calls = 0 + successful_browser_calls = 0 + + for call in context_calls + browser_calls: + call_id = call.get("tool_call_id") + observations = result_map.get(call_id, []) if isinstance(call_id, str) else [] + if not observations: + missing_observations.append(call_id) + continue + if len(observations) != 1: + duplicate_observations.append(call_id) + continue + decoded = _decode_content(observations[0].get("content")) + if _contains_error(decoded) or _contains_error(observations[0]): + error_observations.append(call_id) + continue + if call.get("function_name") == CONTEXT_TOOL: + scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None + context_scope_valid = context_scope_valid and ( + isinstance(scope, dict) + and scope.get("kind") == "project" + and scope.get("project_id") == expected_project_id + ) + successful_context_calls += 1 + elif call.get("function_name") in BROWSER_TOOLS: + successful_browser_calls += 1 + + allowed_missing = allowed_missing_observation_ids or set() + unexpected_missing_observations = [ + call_id for call_id in missing_observations if call_id not in allowed_missing + ] + observations_valid = ( + successful_context_calls > 0 + and successful_browser_calls > 0 + and not (unexpected_missing_observations or duplicate_observations) + ) + direct_http_patterns = re.compile( + r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", + re.IGNORECASE, + ) + direct_http_calls = [ + call + for call in browser_calls + if call.get("function_name") == "mcp__kernel__execute_playwright_code" + and isinstance(call.get("arguments"), dict) + and direct_http_patterns.search(str(call["arguments"].get("code", ""))) + ] + return { + "context_called": bool(context_calls), + "browser_control_called": bool(browser_calls), + "observations_valid": observations_valid, + "context_scope_valid": context_scope_valid and bool(context_calls), + "same_session": same_session, + "no_playwright_mcp": not playwright_calls, + "no_forbidden_kernel_tools": not forbidden_calls, + "no_direct_http_automation": not direct_http_calls, + "missing_observations": missing_observations, + "expected_interrupted_observations": [ + call_id for call_id in missing_observations if call_id in allowed_missing + ], + "unexpected_missing_observations": unexpected_missing_observations, + "duplicate_observations": duplicate_observations, + "error_observations": error_observations, + "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], + "kernel_tool_calls": [ + { + "tool_call_id": call.get("tool_call_id"), + "name": call.get("function_name"), + "arguments": call.get("arguments"), + } + for call in kernel_calls + ], + } + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + trajectory = read_json(LOGS_DIR / "agent" / "trajectory.json") + browser = read_json(Path("/my-info/kernel_browser.json")) + lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) + manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") + clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") + interception = read_json(Path("/data/interception.json")) + agent_stop = read_json(Path("/data/agent-stop.json")) + reward_path = VERIFIER_DIR / "reward.json" + reward_metrics = read_json(reward_path) or {} + + session_id = str((browser or {}).get("session_id") or "") + expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + all_calls = _calls(trajectory or {}) + browser_calls = [call for call in all_calls if call.get("function_name") in BROWSER_TOOLS] + terminal_call_id = browser_calls[-1].get("tool_call_id") if browser_calls else None + stop_detected_at = (agent_stop or {}).get("stop_detected_at") + intercepted_at = (interception or {}).get("intercepted_at") + stopped_after_interception = bool( + isinstance(stop_detected_at, (int, float)) + and isinstance(intercepted_at, (int, float)) + and 0 <= stop_detected_at - intercepted_at <= 5 + ) + allowed_missing = ( + {terminal_call_id} + if stopped_after_interception and isinstance(terminal_call_id, str) + else set() + ) + atif = validate_control( + trajectory, + expected_session_id=session_id, + expected_project_id=expected_project_id, + allowed_missing_observation_ids=allowed_missing, + ) + checks = { + "kernel_mcp_context": atif["context_called"], + "kernel_mcp_browser_control": atif["browser_control_called"], + "kernel_mcp_observations": atif["observations_valid"], + "kernel_mcp_project_scope": atif["context_scope_valid"], + "kernel_mcp_same_session": atif["same_session"], + "no_playwright_mcp": atif["no_playwright_mcp"], + "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], + "no_direct_http_automation": atif["no_direct_http_automation"], + "kernel_mcp_source_sha": bool( + manifest + and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") + ), + "kernel_mcp_manifest_session": bool( + manifest and manifest.get("browser_session_id") == session_id + ), + "kernel_mcp_toolset_allowlist": bool( + manifest + and set(str(manifest.get("enabled_toolsets", "")).split()) + == {"playwright"} + ), + "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), + "browser_deleted": bool( + lifecycle + and lifecycle.get("status") == "deleted" + and lifecycle.get("deletion_verified") is True + ), + "clawbench_intercepted": bool( + reward_metrics.get("intercepted") == 1 + or (clawbench_result or {}).get("intercepted") is True + ), + "agent_stopped_after_interception": stopped_after_interception, + } + infra_ok = all( + value + for name, value in checks.items() + if name not in {"clawbench_intercepted", "agent_stopped_after_interception"} + ) + checks["infra_ok"] = infra_ok + + reward_metrics.update({name: float(value) for name, value in checks.items()}) + reward_path.write_text(json.dumps(reward_metrics, indent=2)) + result = { + "checks": checks, + "session_id": session_id, + "expected_project_id": expected_project_id, + "atif": atif, + "run_manifest": manifest, + "browser_lifecycle": lifecycle, + "clawbench_result": clawbench_result, + "interception": interception, + "agent_stop": agent_stop, + "stop_latency_seconds": ( + stop_detected_at - intercepted_at if stopped_after_interception else None + ), + } + (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile new file mode 100644 index 0000000..9bbc1c8 --- /dev/null +++ b/benchmarks/harbor/image/Dockerfile @@ -0,0 +1,34 @@ +FROM node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + jq \ + procps \ + python3 \ + redis-server \ + && rm -rf /var/lib/apt/lists/* \ + && npm install --global bun@1.3.3 mcp-remote@0.1.38 + +COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv + +WORKDIR /opt/kernel-mcp-server + +COPY package.json bun.lock ./ +RUN bun install --frozen-lockfile + +COPY . . +RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ + KERNEL_CLI_STAGING_CLIENT_ID=kernel-mcp-benchmark \ + KERNEL_CLI_DEV_CLIENT_ID=kernel-mcp-benchmark \ + CLERK_SECRET_KEY=sk_test_kernel_mcp_benchmark_local_only \ + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ \ + bun run build \ + && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ + && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ + && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA + +ENV NEXT_TELEMETRY_DISABLED=1 +WORKDIR /app diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 52061de..a6b0c94 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -83,6 +83,37 @@ describe("MCP Apps additive registration", () => { }); }); +describe("MCP toolset allowlist", () => { + test("keeps connection context and only the selected browser controls", () => { + const previousEnabled = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + const previousDisabled = process.env.KERNEL_MCP_DISABLED_TOOLSETS; + process.env.KERNEL_MCP_ENABLED_TOOLSETS = + "execute_playwright_code computer_action"; + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + try { + const registration = captureRegistration(false); + expect(registration.legacyTools).toEqual([ + "get_connection_context", + "computer_action", + "execute_playwright_code", + ]); + expect(registration.appTools).toEqual([]); + expect(registration.resources).toEqual([]); + } finally { + if (previousEnabled === undefined) { + delete process.env.KERNEL_MCP_ENABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_ENABLED_TOOLSETS = previousEnabled; + } + if (previousDisabled === undefined) { + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_DISABLED_TOOLSETS = previousDisabled; + } + } + }); +}); + describe("project selection registration", () => { const projectScopedTools = [ "manage_profiles", diff --git a/src/lib/mcp/register.ts b/src/lib/mcp/register.ts index 59db2cb..712be6c 100644 --- a/src/lib/mcp/register.ts +++ b/src/lib/mcp/register.ts @@ -90,6 +90,33 @@ function normalizeMcpToolset(value: string): McpToolset | undefined { return undefined; } +function enabledMcpToolsetsFromEnv() { + const raw = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + if (!raw?.trim()) return undefined; + + const enabled = new Set(); + const unknown: string[] = []; + for (const value of raw.split(/[,\s]+/)) { + const token = value.trim().toLowerCase(); + if (!token || token === "none") continue; + if (token === "all") return new Set(mcpToolsets); + + const toolset = normalizeMcpToolset(token); + if (toolset) { + enabled.add(toolset); + } else { + unknown.push(value); + } + } + + if (unknown.length > 0) { + throw new Error( + `Unknown KERNEL_MCP_ENABLED_TOOLSETS value(s): ${unknown.join(", ")}. Supported toolsets: ${mcpToolsets.join(", ")}.`, + ); + } + return enabled; +} + function disabledMcpToolsetsFromEnv() { const raw = process.env.KERNEL_MCP_DISABLED_TOOLSETS; if (!raw?.trim()) return new Set(); @@ -126,10 +153,14 @@ function disabledMcpToolsetsFromEnv() { } function toolsetEnabled( + enabledToolsets: Set | undefined, disabledToolsets: Set, toolset: McpToolset, ) { - return !disabledToolsets.has(toolset); + return ( + (enabledToolsets === undefined || enabledToolsets.has(toolset)) && + !disabledToolsets.has(toolset) + ); } export function registerMcpCapabilities( @@ -139,6 +170,7 @@ export function registerMcpCapabilities( dependencies = defaultMcpDependencies, }: McpRegistrationOptions = {}, ) { + const enabledToolsets = enabledMcpToolsetsFromEnv(); const disabledToolsets = disabledMcpToolsetsFromEnv(); registerKernelPrompts(server); @@ -147,7 +179,7 @@ export function registerMcpCapabilities( registerConnectionContextTool(server); for (const [toolset, registerToolset] of mcpToolRegistrations) { - if (toolsetEnabled(disabledToolsets, toolset)) { + if (toolsetEnabled(enabledToolsets, disabledToolsets, toolset)) { registerToolset(server, dependencies); } } @@ -155,7 +187,10 @@ export function registerMcpCapabilities( // Managed Auth remains fully programmatic for every client. MCP Apps support // adds one interactive launcher (plus its app-only implementation tools and // resource) without replacing or narrowing manage_auth_connections. - if (mcpApps && toolsetEnabled(disabledToolsets, "auth_connections")) { + if ( + mcpApps && + toolsetEnabled(enabledToolsets, disabledToolsets, "auth_connections") + ) { registerAuthLoginApp(server); } } diff --git a/src/lib/mcp/tools/playwright.ts b/src/lib/mcp/tools/playwright.ts index 585bff5..3fd0120 100644 --- a/src/lib/mcp/tools/playwright.ts +++ b/src/lib/mcp/tools/playwright.ts @@ -31,7 +31,7 @@ export function registerPlaywrightTool( code: z .string() .describe( - "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Example: `await page.goto('https://example.com'); return await page.title();` Return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page.", + "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Every invocation should return useful page state. After navigation or interaction, return a condensed accessibility snapshot of the relevant region, e.g. `await page.goto('https://example.com'); return await page.locator('main').ariaSnapshot();` or `await page.getByRole('button', { name: 'Submit' }).click(); return await page.locator('main').ariaSnapshot();`. For targeted reads, return a compact value or object. Do not dump the full DOM or body text.", ), session_id: z .string()