From 8e0cbbd78260e4e481427a6fb53ee5ce2f917d6e Mon Sep 17 00:00:00 2001 From: adidottxt Date: Wed, 12 Aug 2026 14:10:46 -0400 Subject: [PATCH 1/3] chore: add agent app-creation docs evaluation harness --- evals/agent-app-creation/README.md | 51 +++ evals/agent-app-creation/claude-outcomes.mjs | 308 +++++++++++++ evals/agent-app-creation/fixtures/after.json | 17 + evals/agent-app-creation/fixtures/before.json | 29 ++ evals/agent-app-creation/queries.json | 78 ++++ evals/agent-app-creation/retrieval.mjs | 269 +++++++++++ evals/agent-app-creation/retrieval.test.mjs | 71 +++ evals/agent-app-creation/run.mjs | 304 +++++++++++++ evals/agent-app-creation/score.mjs | 420 ++++++++++++++++++ evals/agent-app-creation/score.test.mjs | 294 ++++++++++++ evals/agent-app-creation/visibility.json | 26 ++ 11 files changed, 1867 insertions(+) create mode 100644 evals/agent-app-creation/README.md create mode 100644 evals/agent-app-creation/claude-outcomes.mjs create mode 100644 evals/agent-app-creation/fixtures/after.json create mode 100644 evals/agent-app-creation/fixtures/before.json create mode 100644 evals/agent-app-creation/queries.json create mode 100644 evals/agent-app-creation/retrieval.mjs create mode 100644 evals/agent-app-creation/retrieval.test.mjs create mode 100644 evals/agent-app-creation/run.mjs create mode 100644 evals/agent-app-creation/score.mjs create mode 100644 evals/agent-app-creation/score.test.mjs create mode 100644 evals/agent-app-creation/visibility.json diff --git a/evals/agent-app-creation/README.md b/evals/agent-app-creation/README.md new file mode 100644 index 00000000..22407344 --- /dev/null +++ b/evals/agent-app-creation/README.md @@ -0,0 +1,51 @@ +# Agent app-creation documentation evaluation + +This evaluation measures whether Porter documentation makes the agent path prominent for application creation. The primary harness is local and reproducible: it fetches the live and proposed `llms-full.txt` files, selects each query's explicit canonical entry page, chunks both versions identically, ranks sections within that page with a small fielded BM25 implementation, and compares the context required to discover the agent workflow. + +The primary metric is **tokens to first agent path**: the cumulative estimated documentation tokens before a ranked chunk names `create_app`, or connects the Porter MCP server or an agent to creating or deploying an application. Missing guidance receives one shared before-and-after penalty. The estimate is stable UTF-8 bytes divided by four, so it is suitable for relative comparisons without depending on a vendor tokenizer. The test measures prominence after an agent fetches an app-creation page. + +The complete path requires all of the following facts: + +- Call `create_app`. +- Pass `source` and `build`. +- Install the Porter GitHub App before calling the tool. +- Porter opens a pull request. +- Merge the pull request to trigger the first real deployment. + +## Run the local retrieval evaluation + +The preview must be public and expose `llms-full.txt` and `.md` page exports. + +```bash +node evals/agent-app-creation/run.mjs \ + --after-url https://porter-preview.mintlify.site \ + --json /tmp/porter-agent-app-creation.json \ + --markdown /tmp/porter-agent-app-creation.md +``` + +The report records SHA-256 hashes for both input corpora. It also verifies that the agent-only direction appears in each proposed `.md` export but not in the visible HTML page. + +By default, each answer receives only the top four chunks, creating a small and explicit documentation-context budget while preserving the stricter top-two ranking gates. Use `--max-chunks` to sweep smaller or larger windows when diagnosing a failure. + +## Run the Claude outcome evaluation + +Use the generated retrieval JSON as the only documentation context supplied to local Claude: + +```bash +node evals/agent-app-creation/claude-outcomes.mjs \ + --evaluation /tmp/porter-agent-app-creation.json \ + --json /tmp/porter-agent-app-creation-claude.json \ + --markdown /tmp/porter-agent-app-creation-claude.md +``` + +This performs three trials for each generic and agent-aware query against both corpora, plus three trials for each proposed workflow query. Claude runs in safe mode with tools and project customizations disabled. The report records the estimated retrieved-document tokens and the model envelope's exact total input tokens (including fixed harness overhead), model, context-window size, and cost. + +The outcome gate requires at least 15 of 18 proposed answers to lead with the agent path, an improvement of at least six answers over production, and at least five of six workflow answers to include all five required workflow facts. This remains a manual pull-request gate because model sampling can vary. + +## Test the scorers and retriever + +```bash +node --test \ + evals/agent-app-creation/score.test.mjs \ + evals/agent-app-creation/retrieval.test.mjs +``` diff --git a/evals/agent-app-creation/claude-outcomes.mjs b/evals/agent-app-creation/claude-outcomes.mjs new file mode 100644 index 00000000..1f0d6fdd --- /dev/null +++ b/evals/agent-app-creation/claude-outcomes.mjs @@ -0,0 +1,308 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' + +import { evaluateClaudeOutcomes, median, scoreClaudeAnswer } from './score.mjs' + +const SYSTEM_PROMPT = + 'Follow the user instruction exactly. Use only the supplied documentation context.' + +const usage = `Usage: + node evals/agent-app-creation/claude-outcomes.mjs \\ + --evaluation /tmp/porter-agent-app-creation.json \\ + --json /tmp/porter-agent-app-creation-claude.json \\ + --markdown /tmp/porter-agent-app-creation-claude.md + +Options: + --trials N Trials per query and corpus (default: 3). + --model NAME Optional Claude model override. + --help Show this help text.` + +const parseArgs = (argv) => { + const options = { trials: 3 } + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] + if (argument === '--help') { + options.help = true + continue + } + if (!argument.startsWith('--')) { + throw new Error(`Unexpected argument: ${argument}`) + } + const value = argv[index + 1] + if (!value || value.startsWith('--')) { + throw new Error(`Missing value for ${argument}`) + } + index += 1 + const name = argument + .slice(2) + .replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()) + options[name] = value + } + options.trials = Number(options.trials) + if (!Number.isInteger(options.trials) || options.trials < 1) { + throw new Error('--trials must be a positive integer') + } + return options +} + +const documentationContext = (response) => + (response.infoSnippets ?? []) + .map( + (snippet, index) => + `Snippet ${index + 1}\nSource: ${snippet.pageId}\nSection: ${snippet.breadcrumb ?? 'Untitled'}\n${snippet.content}` + ) + .join('\n\n') + +const evaluationPrompt = ( + query, + response +) => `You are evaluating whether retrieved Porter documentation helps an agent answer a deployment question. + +Answer the user using only the documentation snippets below. Do not use outside knowledge. Recommend the best available interaction path first. Do not discuss this evaluation. + +Return one JSON object and nothing else: +{"leadPath":"agent|dashboard|cli|unclear","answer":"your concise answer"} + +Use leadPath "agent" only when the answer recommends an AI or coding agent using Porter's MCP server as the first path. + +User question: +${query.prompt} + +Documentation snippets: +${documentationContext(response)}` + +const parseClaudeResult = (stdout) => { + const envelope = JSON.parse(stdout) + const result = + typeof envelope.result === 'string' + ? envelope.result.trim() + : envelope.result + const parsed = + typeof result === 'object' && result + ? result + : JSON.parse( + String(result) + .replace(/^```(?:json)?\s*/i, '') + .replace(/\s*```$/, '') + ) + if (!['agent', 'dashboard', 'cli', 'unclear'].includes(parsed.leadPath)) { + throw new Error(`Claude returned an invalid leadPath: ${parsed.leadPath}`) + } + if (typeof parsed.answer !== 'string' || parsed.answer.length === 0) { + throw new Error('Claude returned an empty answer') + } + const modelUsage = Object.entries(envelope.modelUsage ?? {}) + const primaryModel = + modelUsage.find( + ([, usage]) => usage.outputTokens === envelope.usage?.output_tokens + ) ?? + modelUsage.sort(([, left], [, right]) => right.costUSD - left.costUSD)[0] + const directUsage = envelope.usage ?? {} + const effectiveInputTokens = + (directUsage.input_tokens ?? 0) + + (directUsage.cache_creation_input_tokens ?? 0) + + (directUsage.cache_read_input_tokens ?? 0) + return { + answer: parsed, + model: primaryModel?.[0] ?? envelope.model ?? 'unknown', + usage: { + envelopeInputTokens: + effectiveInputTokens || primaryModel?.[1].inputTokens, + uncachedInputTokens: directUsage.input_tokens, + cacheCreationInputTokens: directUsage.cache_creation_input_tokens, + cacheReadInputTokens: directUsage.cache_read_input_tokens, + outputTokens: + envelope.usage?.output_tokens ?? primaryModel?.[1].outputTokens, + contextWindow: primaryModel?.[1].contextWindow, + costUSD: envelope.total_cost_usd + } + } +} + +const runClaude = (prompt, model) => + new Promise((resolve, reject) => { + const arguments_ = [ + '-p', + '--output-format', + 'json', + '--no-session-persistence', + '--tools', + '', + '--safe-mode', + '--system-prompt', + SYSTEM_PROMPT, + '--disable-slash-commands' + ] + if (model) { + arguments_.push('--model', model) + } + const child = spawn('claude', arguments_, { + stdio: ['pipe', 'pipe', 'pipe'] + }) + let stdout = '' + let stderr = '' + const timeout = setTimeout(() => { + child.kill('SIGTERM') + reject(new Error('Claude evaluation timed out after 120 seconds')) + }, 120_000) + + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + child.stderr.on('data', (chunk) => { + stderr += chunk + }) + child.on('error', (error) => { + clearTimeout(timeout) + reject(error) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (code !== 0) { + reject(new Error(`Claude exited with ${code}: ${stderr.trim()}`)) + return + } + try { + resolve(parseClaudeResult(stdout)) + } catch (error) { + reject( + new Error( + `Could not parse Claude output: ${error.message}\n${stdout.slice(0, 500)}` + ) + ) + } + }) + child.stdin.end(prompt) + }) + +const buildJobs = (evaluation, trials) => { + const jobs = [] + for (const { query } of evaluation.comparison.results) { + const corpora = ['generic', 'agent-aware'].includes(query.group) + ? ['before', 'after'] + : query.group === 'workflow' + ? ['after'] + : [] + for (const corpus of corpora) { + for (let trial = 1; trial <= trials; trial += 1) { + jobs.push({ + corpus, + query, + response: evaluation[corpus].responses[query.id], + trial + }) + } + } + } + return jobs +} + +const renderReport = (outcome) => { + const lines = [ + '# Claude outcome evaluation', + '', + `Result: **${outcome.pass ? 'PASS' : 'FAIL'}**`, + `Models: ${[...new Set(outcome.runs.map((run) => run.model))].join(', ')}`, + `Median retrieved context: before ${outcome.usageSummary.before.medianRetrievedContextTokens} estimated tokens; after ${outcome.usageSummary.after.medianRetrievedContextTokens} estimated tokens`, + `Median total model-envelope input (including harness overhead): before ${outcome.usageSummary.before.medianEnvelopeInputTokens} tokens; after ${outcome.usageSummary.after.medianEnvelopeInputTokens} tokens`, + '', + '| Gate | Result | Detail |', + '| --- | --- | --- |', + ...outcome.gates.map( + (gate) => + `| ${gate.id} | ${gate.pass ? 'PASS' : 'FAIL'} | ${gate.detail} |` + ), + '', + '| Corpus | Query | Trial | Retrieved context | Envelope input | Reported lead | Inferred lead | Source + build | Generated PR | GitHub App prerequisite | Merge step |', + '| --- | --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: | ---: |', + ...outcome.runs.map((run) => { + const score = scoreClaudeAnswer(run.answer) + return `| ${run.corpus} | ${run.queryId} | ${run.trial} | ${run.retrievedContextTokens} | ${run.usage.envelopeInputTokens} | ${run.answer.leadPath} | ${score.inferredLeadPath} | ${score.mentionsSourceAndBuild ? 'yes' : 'no'} | ${score.mentionsGeneratedPullRequest ? 'yes' : 'no'} | ${score.mentionsGithubAppPrerequisite ? 'yes' : 'no'} | ${score.mentionsMergeStep ? 'yes' : 'no'} |` + }) + ] + return `${lines.join('\n')}\n` +} + +const writeOutput = async (path, content) => { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) +} + +const main = async () => { + const options = parseArgs(process.argv.slice(2)) + if (options.help) { + process.stdout.write(`${usage}\n`) + return + } + if (!options.evaluation) { + throw new Error(`Missing required option --evaluation\n\n${usage}`) + } + + const evaluation = JSON.parse(await readFile(options.evaluation, 'utf8')) + const runs = [] + for (const job of buildJobs(evaluation, options.trials)) { + process.stderr.write(`${job.corpus} ${job.query.id} trial ${job.trial}\n`) + const { answer, model, usage } = await runClaude( + evaluationPrompt(job.query, job.response), + options.model + ) + runs.push({ + corpus: job.corpus, + queryId: job.query.id, + queryGroup: job.query.group, + trial: job.trial, + model, + usage, + retrievedContextTokens: (job.response.infoSnippets ?? []).reduce( + (total, snippet) => total + snippet.contentTokens, + 0 + ), + answer + }) + } + const deploymentRuns = runs.filter(({ queryGroup }) => + ['generic', 'agent-aware'].includes(queryGroup) + ) + const usageSummary = Object.fromEntries( + ['before', 'after'].map((corpus) => { + const corpusRuns = deploymentRuns.filter((run) => run.corpus === corpus) + return [ + corpus, + { + medianRetrievedContextTokens: median( + corpusRuns.map( + ({ retrievedContextTokens }) => retrievedContextTokens + ) + ), + medianEnvelopeInputTokens: median( + corpusRuns.map(({ usage }) => usage.envelopeInputTokens) + ) + } + ] + }) + ) + const outcome = { + generatedAt: new Date().toISOString(), + usageSummary, + ...evaluateClaudeOutcomes(runs) + } + const report = renderReport(outcome) + if (options.json) { + await writeOutput(options.json, `${JSON.stringify(outcome, null, 2)}\n`) + } + if (options.markdown) { + await writeOutput(options.markdown, report) + } + process.stdout.write(report) + if (!outcome.pass) { + process.exitCode = 1 + } +} + +main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`) + process.exitCode = 1 +}) diff --git a/evals/agent-app-creation/fixtures/after.json b/evals/agent-app-creation/fixtures/after.json new file mode 100644 index 00000000..6df2d1a0 --- /dev/null +++ b/evals/agent-app-creation/fixtures/after.json @@ -0,0 +1,17 @@ +{ + "codeSnippets": [], + "infoSnippets": [ + { + "pageId": "https://docs.porter.run/getting-started/quickstart", + "breadcrumb": "Create your first application > Agent", + "content": "Connect your agent to the Porter MCP server and install the Porter GitHub App before starting. The agent calls create_app with source and build. Porter creates the application on a placeholder image and opens a pull request. Merge the PR to trigger the first real deployment.", + "contentTokens": 40 + }, + { + "pageId": "https://docs.porter.run/applications/deploy/deploy-from-github-repo", + "breadcrumb": "Customizing your deployment", + "content": "You can customize the build configuration and each application service.", + "contentTokens": 20 + } + ] +} diff --git a/evals/agent-app-creation/fixtures/before.json b/evals/agent-app-creation/fixtures/before.json new file mode 100644 index 00000000..3bb309a7 --- /dev/null +++ b/evals/agent-app-creation/fixtures/before.json @@ -0,0 +1,29 @@ +{ + "codeSnippets": [], + "infoSnippets": [ + { + "pageId": "https://docs.porter.run/getting-started/quickstart", + "breadcrumb": "Create your first application > Connect GitHub", + "content": "Open the Porter dashboard, click Create Application, and connect the repository.", + "contentTokens": 25 + }, + { + "pageId": "https://docs.porter.run/getting-started/quickstart", + "breadcrumb": "Create your first application > Select repository", + "content": "Choose the repository and branch that you want Porter to deploy.", + "contentTokens": 25 + }, + { + "pageId": "https://docs.porter.run/applications/deploy/deploy-from-github-repo", + "breadcrumb": "Review detected applications", + "content": "Review the applications detected from your repository in the dashboard.", + "contentTokens": 25 + }, + { + "pageId": "https://docs.porter.run/applications/deploy/deploy-from-github-repo", + "breadcrumb": "Deploy", + "content": "Click Deploy when the configuration looks correct.", + "contentTokens": 25 + } + ] +} diff --git a/evals/agent-app-creation/queries.json b/evals/agent-app-creation/queries.json new file mode 100644 index 00000000..cc0ce569 --- /dev/null +++ b/evals/agent-app-creation/queries.json @@ -0,0 +1,78 @@ +{ + "version": 1, + "queries": [ + { + "id": "generic-create-github", + "group": "generic", + "entryPath": "/applications/deploy/deploy-from-github-repo", + "prompt": "How do I create a Porter application from a GitHub repository?" + }, + { + "id": "generic-deploy-repository", + "group": "generic", + "entryPath": "/applications/deploy/deploy-from-github-repo", + "prompt": "Deploy my GitHub repository to Porter." + }, + { + "id": "generic-first-application", + "group": "generic", + "entryPath": "/getting-started/quickstart", + "prompt": "What is the fastest way to create my first application on Porter?" + }, + { + "id": "generic-create-from-source", + "group": "generic", + "entryPath": "/applications/deploy/overview", + "prompt": "How should I create a Porter application from source code?" + }, + { + "id": "agent-coding-agent", + "group": "agent-aware", + "entryPath": "/mcp/overview", + "prompt": "Can my coding agent create a Porter application for me?" + }, + { + "id": "agent-mcp", + "group": "agent-aware", + "entryPath": "/mcp/tools", + "prompt": "How do I create an application using the Porter MCP server?" + }, + { + "id": "workflow-prerequisites", + "group": "workflow", + "entryPath": "/applications/deploy/deploy-from-github-repo", + "prompt": "What setup is required before my agent creates a Porter application from a GitHub repository?" + }, + { + "id": "workflow-after-create", + "group": "workflow", + "entryPath": "/mcp/tools", + "prompt": "What happens after an agent creates a Porter application from source?" + }, + { + "id": "control-prebuilt-dashboard", + "group": "control", + "entryPath": "/applications/deploy/deploy-from-docker-registry", + "prompt": "How do I deploy a prebuilt container image with the Porter dashboard?", + "expectedTopTwoTerms": [ + ["dashboard"], + [ + "prebuilt image", + "container image", + "container registry", + "docker registry" + ] + ] + }, + { + "id": "control-customization", + "group": "control", + "entryPath": "/applications/deploy/deploy-from-github-repo", + "prompt": "How do I customize a Porter application's build method and services?", + "expectedTopTwoTerms": [ + ["build method", "build configuration", "build"], + ["service", "web service", "worker", "job"] + ] + } + ] +} diff --git a/evals/agent-app-creation/retrieval.mjs b/evals/agent-app-creation/retrieval.mjs new file mode 100644 index 00000000..4d60ebad --- /dev/null +++ b/evals/agent-app-creation/retrieval.mjs @@ -0,0 +1,269 @@ +const DEFAULT_CHUNK_TOKENS = 600 + +const STOP_WORDS = new Set([ + 'a', + 'an', + 'and', + 'can', + 'do', + 'for', + 'from', + 'how', + 'i', + 'is', + 'it', + 'my', + 'of', + 'on', + 'the', + 'to', + 'using', + 'what', + 'with' +]) + +const canonicalToken = (token) => { + const aliases = { + applications: 'application', + apps: 'application', + customization: 'customize', + customized: 'customize', + customizing: 'customize', + created: 'create', + creates: 'create', + creating: 'create', + creation: 'create', + deployed: 'deploy', + deploying: 'deploy', + deployment: 'deploy', + deployments: 'deploy', + repositories: 'repository', + repos: 'repository' + } + return aliases[token] ?? token +} + +const searchTokens = (text) => + (text.toLowerCase().match(/[a-z0-9_]+/g) ?? []) + .map(canonicalToken) + .filter((token) => token.length > 1 && !STOP_WORDS.has(token)) + +export const estimateTokens = (text) => + Math.max(1, Math.ceil(Buffer.byteLength(text, 'utf8') / 4)) + +const splitOversizedParagraph = (paragraph, maxTokens) => { + const maximumBytes = maxTokens * 4 + const pieces = [] + let remaining = paragraph + while (Buffer.byteLength(remaining, 'utf8') > maximumBytes) { + let end = 0 + let bytes = 0 + for (const character of remaining) { + const characterBytes = Buffer.byteLength(character, 'utf8') + if (bytes + characterBytes > maximumBytes) break + bytes += characterBytes + end += character.length + } + const boundary = Math.max( + remaining.lastIndexOf('\n', end), + remaining.lastIndexOf(' ', end) + ) + if (boundary > end / 2) end = boundary + const piece = remaining.slice(0, end).trim() + if (piece) pieces.push(piece) + remaining = remaining.slice(end).trim() + } + if (remaining) pieces.push(remaining) + return pieces +} + +const sectionUnits = (content) => { + const units = [] + const visibilityPattern = /]*>[\s\S]*?<\/Visibility>/g + let cursor = 0 + const addParagraphs = (text) => { + units.push( + ...text + .trim() + .split(/\n{2,}/) + .filter(Boolean) + .map((content) => ({ content, atomic: false })) + ) + } + + for (const match of content.matchAll(visibilityPattern)) { + addParagraphs(content.slice(cursor, match.index)) + units.push({ content: match[0].trim(), atomic: true }) + cursor = match.index + match[0].length + } + addParagraphs(content.slice(cursor)) + return units +} + +const chunkSection = (section, maxTokens) => { + const paragraphs = sectionUnits(section.content).flatMap( + ({ content, atomic }) => + estimateTokens(content) > maxTokens && !atomic + ? splitOversizedParagraph(content, maxTokens) + : content + ) + const chunks = [] + let current = [] + + for (const paragraph of paragraphs) { + const candidate = [...current, paragraph].join('\n\n') + if (current.length > 0 && estimateTokens(candidate) > maxTokens) { + chunks.push(current.join('\n\n')) + current = [paragraph] + } else { + current.push(paragraph) + } + } + if (current.length > 0) chunks.push(current.join('\n\n')) + return chunks +} + +const pageSections = (body) => { + const headings = [...body.matchAll(/^## (.+)$/gm)] + const sections = [] + const introductionEnd = headings[0]?.index ?? body.length + const introduction = body.slice(0, introductionEnd).trim() + if (introduction) { + sections.push({ title: 'Introduction', content: introduction }) + } + for (const [index, heading] of headings.entries()) { + const end = headings[index + 1]?.index ?? body.length + sections.push({ + title: heading[1].trim(), + content: body.slice(heading.index, end).trim() + }) + } + return sections +} + +export const parseCorpus = ( + corpus, + { maxChunkTokens = DEFAULT_CHUNK_TOKENS } = {} +) => { + const headers = [...corpus.matchAll(/^# (.+)\nSource: (https?:\/\/\S+)\n/gm)] + if (headers.length === 0) { + throw new Error('The llms-full.txt corpus contains no page headers') + } + + const chunks = [] + for (const [pageIndex, header] of headers.entries()) { + const bodyStart = header.index + header[0].length + const bodyEnd = headers[pageIndex + 1]?.index ?? corpus.length + const pageTitle = header[1].trim() + const pageId = header[2].trim() + const sections = pageSections(corpus.slice(bodyStart, bodyEnd)) + + for (const [sectionIndex, section] of sections.entries()) { + for (const [chunkIndex, content] of chunkSection( + section, + maxChunkTokens + ).entries()) { + chunks.push({ + pageId, + pageTitle, + sectionTitle: section.title, + breadcrumb: `${pageTitle} > ${section.title}`, + content, + contentTokens: estimateTokens(content), + sourceOrder: [pageIndex, sectionIndex, chunkIndex] + }) + } + } + } + return chunks +} + +const termFrequencies = (tokens) => { + const frequencies = new Map() + for (const token of tokens) { + frequencies.set(token, (frequencies.get(token) ?? 0) + 1) + } + return frequencies +} + +export const rankChunks = (chunks, query) => { + const terms = [...new Set(searchTokens(query))] + const querySequence = searchTokens(query) + const queryBigrams = querySequence + .slice(0, -1) + .map((token, index) => `${token} ${querySequence[index + 1]}`) + const documents = chunks.map((chunk) => { + const contentTokens = searchTokens(chunk.content) + const titleTokens = searchTokens(chunk.pageTitle) + const sectionTokens = searchTokens(chunk.sectionTitle) + return { + chunk, + contentTokens, + contentFrequencies: termFrequencies(contentTokens), + titleFrequencies: termFrequencies(titleTokens), + sectionFrequencies: termFrequencies(sectionTokens), + headingText: [...titleTokens, ...sectionTokens].join(' '), + allTerms: new Set([...contentTokens, ...titleTokens, ...sectionTokens]) + } + }) + const averageLength = + documents.reduce( + (total, document) => total + document.contentTokens.length, + 0 + ) / documents.length + const documentFrequency = new Map( + terms.map((token) => [ + token, + documents.filter(({ allTerms }) => allTerms.has(token)).length + ]) + ) + const k1 = 1.2 + const b = 0.75 + + return documents + .map((document) => { + const score = terms.reduce((total, token) => { + const contentFrequency = document.contentFrequencies.get(token) ?? 0 + const titleFrequency = document.titleFrequencies.get(token) ?? 0 + const sectionFrequency = document.sectionFrequencies.get(token) ?? 0 + if (contentFrequency + titleFrequency + sectionFrequency === 0) { + return total + } + const containingDocuments = documentFrequency.get(token) + const inverseDocumentFrequency = Math.log( + 1 + + (documents.length - containingDocuments + 0.5) / + (containingDocuments + 0.5) + ) + const normalizedFrequency = + (contentFrequency * (k1 + 1)) / + (contentFrequency + + k1 * (1 - b + b * (document.contentTokens.length / averageLength))) + const fieldFrequency = titleFrequency * 3 + sectionFrequency * 3 + return ( + total + + inverseDocumentFrequency * (normalizedFrequency + fieldFrequency) + ) + }, 0) + const phraseScore = queryBigrams.filter((bigram) => + document.headingText.includes(bigram) + ).length + return { + ...document.chunk, + retrievalScore: score + phraseScore * 2 + } + }) + .sort((left, right) => { + if (right.retrievalScore !== left.retrievalScore) { + return right.retrievalScore - left.retrievalScore + } + for (let index = 0; index < left.sourceOrder.length; index += 1) { + const difference = left.sourceOrder[index] - right.sourceOrder[index] + if (difference !== 0) return difference + } + return 0 + }) +} + +export const retrieve = (chunks, query, maximumChunks) => + rankChunks(chunks, query).slice(0, maximumChunks) diff --git a/evals/agent-app-creation/retrieval.test.mjs b/evals/agent-app-creation/retrieval.test.mjs new file mode 100644 index 00000000..4c9f4e6d --- /dev/null +++ b/evals/agent-app-creation/retrieval.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { estimateTokens, parseCorpus, rankChunks } from './retrieval.mjs' + +const corpus = `# Dashboard deployment +Source: https://docs.example.com/dashboard + +## Create an application + +Open the dashboard and select a GitHub repository. + +# Agent deployment +Source: https://docs.example.com/agent + +## Create an application + +Use the Porter MCP server and call \`create_app\` with source and build. +` + +test('parses llms-full pages into bounded chunks with source metadata', () => { + const chunks = parseCorpus(corpus, { maxChunkTokens: 20 }) + assert.ok(chunks.length >= 2) + assert.equal(chunks[0].pageId, 'https://docs.example.com/dashboard') + assert.ok(chunks.every(({ contentTokens }) => contentTokens > 0)) +}) + +test('ranks the agent chunk first for an MCP application query', () => { + const chunks = parseCorpus(corpus) + const ranked = rankChunks( + chunks, + 'Can my coding agent create an application with the Porter MCP server?' + ) + assert.equal(ranked[0].pageId, 'https://docs.example.com/agent') + assert.match(ranked[0].content, /create_app/) +}) + +test('uses a deterministic UTF-8 token estimate', () => { + assert.equal(estimateTokens('12345678'), 2) +}) + +test('keeps multibyte chunks within the configured token estimate', () => { + const content = '🚀'.repeat(100) + const unicodeCorpus = `# Unicode +Source: https://docs.example.com/unicode + +${content} +` + const chunks = parseCorpus(unicodeCorpus, { maxChunkTokens: 10 }) + assert.ok(chunks.length > 1) + assert.ok(chunks.every(({ contentTokens }) => contentTokens <= 10)) + assert.equal(chunks.map((chunk) => chunk.content).join(''), content) +}) + +test('keeps multi-paragraph agent-only visibility guidance atomic', () => { + const visibilityCorpus = `# Agent +Source: https://docs.example.com/agent + + +Call create_app with source and build. + +${'The user merges the generated pull request. '.repeat(20)} + +` + const chunks = parseCorpus(visibilityCorpus, { maxChunkTokens: 10 }) + const guidance = chunks.filter(({ content }) => /create_app/.test(content)) + assert.equal(guidance.length, 1) + assert.match(guidance[0].content, //) + assert.match(guidance[0].content, /generated pull request/) + assert.match(guidance[0].content, /<\/Visibility>/) +}) diff --git a/evals/agent-app-creation/run.mjs b/evals/agent-app-creation/run.mjs new file mode 100644 index 00000000..cff59a79 --- /dev/null +++ b/evals/agent-app-creation/run.mjs @@ -0,0 +1,304 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { parseCorpus, retrieve } from './retrieval.mjs' +import { compareResults } from './score.mjs' + +const DEFAULT_BEFORE_URL = 'https://docs.porter.run' + +const usage = `Usage: + node evals/agent-app-creation/run.mjs \\ + --after-url https://porter-preview.mintlify.site \\ + --json /tmp/porter-agent-app-creation.json \\ + --markdown /tmp/porter-agent-app-creation.md + +Options: + --before-url URL Live documentation origin (default: https://docs.porter.run). + --after-url URL Proposed documentation origin (required). + --max-chunks N Ranked chunks returned per query (default: 4). + --skip-visibility Skip agent-only Markdown/HTML visibility checks. + --json PATH Write the machine-readable result. + --markdown PATH Write the Markdown report. + --help Show this help text.` + +const parseArgs = (argv) => { + const options = { + beforeUrl: DEFAULT_BEFORE_URL, + maxChunks: 4, + verifyVisibility: true + } + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] + if (argument === '--help') { + options.help = true + continue + } + if (argument === '--skip-visibility') { + options.verifyVisibility = false + continue + } + if (!argument.startsWith('--')) { + throw new Error(`Unexpected argument: ${argument}`) + } + const value = argv[index + 1] + if (!value || value.startsWith('--')) { + throw new Error(`Missing value for ${argument}`) + } + index += 1 + const name = argument + .slice(2) + .replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()) + options[name] = value + } + options.maxChunks = Number(options.maxChunks) + if (!Number.isInteger(options.maxChunks) || options.maxChunks < 1) { + throw new Error('--max-chunks must be a positive integer') + } + return options +} + +const sleep = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)) + +const request = async (url, { attempts = 3 } = {}) => { + let lastError + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }) + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText} from ${url}`) + } + return await response.text() + } catch (error) { + lastError = error + if (attempt < attempts) await sleep(attempt * 1_000) + } + } + throw lastError +} + +const origin = (url) => url.replace(/\/$/, '') + +const corpusMetadata = (sourceUrl, corpus, chunks) => ({ + id: `${origin(sourceUrl)}/llms-full.txt`, + state: 'fetched', + fetchedAt: new Date().toISOString(), + sha256: createHash('sha256').update(corpus).digest('hex'), + pages: new Set(chunks.map(({ pageId }) => pageId)).size, + chunks: chunks.length, + totalTokens: chunks.reduce( + (total, { contentTokens }) => total + contentTokens, + 0 + ) +}) + +const chunksForEntryPath = (chunks, entryPath) => { + const matching = chunks.filter( + ({ pageId }) => new URL(pageId).pathname === entryPath + ) + if (matching.length === 0) { + throw new Error(`No llms-full.txt chunks found for ${entryPath}`) + } + return matching +} + +const responsesFor = (chunks, queries, maximumChunks) => + Object.fromEntries( + queries.map((query) => [ + query.id, + { + infoSnippets: retrieve( + chunksForEntryPath(chunks, query.entryPath), + query.prompt, + maximumChunks + ).map( + ({ pageId, breadcrumb, content, contentTokens, retrievalScore }) => ({ + pageId, + breadcrumb, + content, + contentTokens, + retrievalScore + }) + ), + codeSnippets: [] + } + ]) + ) + +const decodeHtml = (value) => + value + .replace(/"/gi, '"') + .replace(/&#(?:39|x27);/gi, "'") + .replace(/&(?:nbsp|#160|#xA0);/gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + +const visibleText = (html) => + decodeHtml( + html + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/<[^>]+>/g, ' ') + ) + .replace(/[`*_]/g, '') + .replace(/\s+/g, ' ') + .trim() + +const normalizedDocumentation = (content) => + visibleText(content).replace(/[“”]/g, '"').replace(/[‘’]/g, "'") + +const verifyVisibility = async (afterUrl, checks) => { + const baseUrl = origin(afterUrl) + return Promise.all( + checks.map(async (check) => { + const [markdown, html] = await Promise.all([ + request(`${baseUrl}${check.path}.md`), + request(`${baseUrl}${check.path}`) + ]) + const marker = normalizedDocumentation(check.marker) + const markdownIncludesMarker = + normalizedDocumentation(markdown).includes(marker) + const htmlExcludesMarker = !visibleText(html).includes(marker) + return { + ...check, + markdownIncludesMarker, + htmlExcludesMarker, + pass: markdownIncludesMarker && htmlExcludesMarker + } + }) + ) +} + +const metric = (score, valueName, indexName) => + score[indexName] === -1 + ? `not found (${score[valueName]} penalty)` + : String(score[valueName]) + +const renderReport = (evaluation) => { + const lines = [ + '# Porter agent app-creation local retrieval evaluation', + '', + `Result: **${evaluation.pass ? 'PASS' : 'FAIL'}**`, + '', + 'The primary run uses deterministic local chunking and fielded BM25 ranking within each query’s canonical entry page from the public `llms-full.txt`. Token counts are a stable UTF-8 estimate; the companion Claude run records the model envelope’s exact input-token usage.', + '', + `- Before: ${evaluation.before.library.id}`, + `- Before corpus: ${evaluation.before.library.pages} pages, ${evaluation.before.library.chunks} chunks, ${evaluation.before.library.totalTokens} estimated tokens, SHA-256 \`${evaluation.before.library.sha256}\``, + `- After: ${evaluation.after.library.id}`, + `- After corpus: ${evaluation.after.library.pages} pages, ${evaluation.after.library.chunks} chunks, ${evaluation.after.library.totalTokens} estimated tokens, SHA-256 \`${evaluation.after.library.sha256}\``, + `- Returned context: top ${evaluation.maximumChunks} chunks per query`, + `- Visibility checks: ${evaluation.visibilityStatus}`, + '', + '## Gates', + '', + '| Gate | Result | Detail |', + '| --- | --- | --- |', + ...evaluation.comparison.gates.map( + (gate) => + `| ${gate.id} | ${gate.pass ? 'PASS' : 'FAIL'} | ${gate.detail} |` + ), + '', + '## Query results', + '', + '| Group | Query | Entry page | Before first agent rank | After first agent rank | Before tokens to agent | After tokens to agent | After complete path |', + '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |', + ...evaluation.comparison.results.map( + ({ query, before, after }) => + `| ${query.group} | ${query.prompt} | ${query.entryPath} | ${before.firstAgentIndex === -1 ? 'not found' : before.firstAgentIndex + 1} | ${after.firstAgentIndex === -1 ? 'not found' : after.firstAgentIndex + 1} | ${metric(before, 'tokensToFirstAgentPath', 'firstAgentIndex')} | ${metric(after, 'tokensToFirstAgentPath', 'firstAgentIndex')} | ${after.completePath ? 'yes' : 'no'} |` + ) + ] + + if (evaluation.visibility) { + lines.push( + '', + '## Agent visibility', + '', + '| Page | `.md` includes guidance | HTML excludes guidance | Result |', + '| --- | ---: | ---: | ---: |', + ...evaluation.visibility.map( + (check) => + `| ${check.path} | ${check.markdownIncludesMarker ? 'yes' : 'no'} | ${check.htmlExcludesMarker ? 'yes' : 'no'} | ${check.pass ? 'PASS' : 'FAIL'} |` + ) + ) + } + + return `${lines.join('\n')}\n` +} + +const writeOutput = async (path, content) => { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) +} + +const main = async () => { + const options = parseArgs(process.argv.slice(2)) + if (options.help) { + process.stdout.write(`${usage}\n`) + return + } + if (!options.afterUrl) { + throw new Error(`Missing required option --after-url\n\n${usage}`) + } + + const directory = dirname(fileURLToPath(import.meta.url)) + const { queries } = JSON.parse( + await readFile(join(directory, 'queries.json'), 'utf8') + ) + const visibilityChecks = JSON.parse( + await readFile(join(directory, 'visibility.json'), 'utf8') + ) + const [beforeCorpus, afterCorpus] = await Promise.all([ + request(`${origin(options.beforeUrl)}/llms-full.txt`), + request(`${origin(options.afterUrl)}/llms-full.txt`) + ]) + const beforeChunks = parseCorpus(beforeCorpus) + const afterChunks = parseCorpus(afterCorpus) + const beforeResponses = responsesFor(beforeChunks, queries, options.maxChunks) + const afterResponses = responsesFor(afterChunks, queries, options.maxChunks) + const comparison = compareResults(queries, beforeResponses, afterResponses) + const visibility = options.verifyVisibility + ? await verifyVisibility(options.afterUrl, visibilityChecks) + : undefined + const visibilityPass = visibility?.every(({ pass }) => pass) ?? false + const visibilityStatus = options.verifyVisibility + ? visibilityPass + ? 'passed' + : 'failed' + : 'skipped (overall result cannot pass)' + const evaluation = { + generatedAt: new Date().toISOString(), + method: 'local-bm25', + pass: comparison.pass && visibilityPass, + maximumChunks: options.maxChunks, + visibilityStatus, + before: { + sourceUrl: options.beforeUrl, + library: corpusMetadata(options.beforeUrl, beforeCorpus, beforeChunks), + responses: beforeResponses + }, + after: { + sourceUrl: options.afterUrl, + library: corpusMetadata(options.afterUrl, afterCorpus, afterChunks), + responses: afterResponses + }, + comparison, + visibility + } + const report = renderReport(evaluation) + if (options.json) { + await writeOutput(options.json, `${JSON.stringify(evaluation, null, 2)}\n`) + } + if (options.markdown) await writeOutput(options.markdown, report) + process.stdout.write(report) + if (!evaluation.pass) process.exitCode = 1 +} + +main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`) + process.exitCode = 1 +}) diff --git a/evals/agent-app-creation/score.mjs b/evals/agent-app-creation/score.mjs new file mode 100644 index 00000000..61700901 --- /dev/null +++ b/evals/agent-app-creation/score.mjs @@ -0,0 +1,420 @@ +const CREATE_APP_PATTERN = /\bcreate_app\b/i +const AGENT_SIGNAL_PATTERNS = [ + /\bMCP\b/i, + /\b(?:AI|coding) agent\b/i, + /\byour agent\b/i +] + +const sentences = (text) => text.split(/[.!?\n]+/).filter(Boolean) + +const mentionsGithubAppPrerequisite = (text) => + sentences(text).some( + (sentence) => + /\bPorter GitHub App\b/i.test(sentence) && + /\b(?:install(?:ed|ing|ation)?|prerequisite|required|must|before)\b/i.test( + sentence + ) && + !/\b(?:not required|optional|does not need|doesn't need|need not)\b/i.test( + sentence + ) + ) + +const mentionsMergeStep = (text) => + sentences(text).some( + (sentence) => + /\bmerg(?:e|es|ed|ing)\b/i.test(sentence) && + /\b(?:pull request|PR)\b/i.test(sentence) && + !/\b(?:do not|don't|should not|shouldn't|must not|cannot|can't)\b/i.test( + sentence + ) + ) + +const mentionsSourceAndBuild = (text) => + sentences(text).some( + (sentence) => /\bsource\b/i.test(sentence) && /\bbuild\b/i.test(sentence) + ) + +const mentionsGeneratedPullRequest = (text) => + sentences(text).some( + (sentence) => + /\b(?:Porter|create_app|MCP server|agent|tool|it)\b[^.!?\n]{0,100}\b(?:open(?:s|ed|ing)?|creat(?:e|es|ed|ing)|generat(?:e|es|ed|ing)|return(?:s|ed|ing)?)\b[^.!?\n]{0,100}\b(?:pull request|PR)\b/i.test( + sentence + ) && + !/\b(?:ask|tell|instruct)(?:s|ed|ing)?\b[^.!?\n]{0,40}\b(?:you|user)\b|\b(?:open|create|generate)(?:s|d|ed|ing)?\b[^.!?\n]{0,40}\b(?:yourself|manually)\b/i.test( + sentence + ) + ) + +const REQUIRED_FACTS = { + createApp: (text) => /\bcreate_app\b/i.test(text), + sourceAndBuild: (text) => /\bsource\b/i.test(text) && /\bbuild\b/i.test(text), + githubApp: (text) => mentionsGithubAppPrerequisite(text), + pullRequest: (text) => /\bpull request\b|\bPR\b/i.test(text), + merge: (text) => mentionsMergeStep(text) +} + +const FALSE_GITHUB_INSTALL_CLAIMS = [ + /\b(?:MCP server|create_app|agent)\s+(?:itself\s+)?(?:(?:can|will|automatically)\s+)?install(?:s)?\b[^.!?\n]{0,80}\b(?:Porter\s+)?GitHub App\b/i, + /\binstall(?:s|ed|ing)?\b[^.!?\n]{0,80}\b(?:Porter\s+)?GitHub App\b[^.!?\n]{0,120}\b(?:with|using|through)\s+(?:the\s+)?(?:MCP|create_app|agent)\b/i +] + +const snippetText = (snippet) => + [snippet.breadcrumb, snippet.content].filter(Boolean).join('\n') + +const snippetTokens = (snippet) => { + if (!Number.isFinite(snippet.contentTokens) || snippet.contentTokens < 0) { + throw new Error( + `Snippet is missing a valid contentTokens value: ${snippet.pageId ?? 'unknown page'}` + ) + } + + return snippet.contentTokens +} + +const cumulativeTokensThrough = (snippets, index) => + snippets + .slice(0, index + 1) + .reduce((total, snippet) => total + snippetTokens(snippet), 0) + +const containsAgentPath = (text) => + CREATE_APP_PATTERN.test(text) || + (AGENT_SIGNAL_PATTERNS.some((pattern) => pattern.test(text)) && + /\b(?:creat\w*|deploy\w*)\b/i.test(text) && + /\b(?:app|application)s?\b/i.test(text)) + +const factsIn = (text) => + Object.fromEntries( + Object.entries(REQUIRED_FACTS).map(([name, matches]) => [ + name, + matches(text) + ]) + ) + +const allFactsPresent = (facts) => Object.values(facts).every(Boolean) + +const expectedTermsPresent = (snippets, expectedTermGroups = []) => { + const topTwoText = snippets + .slice(0, 2) + .map(snippetText) + .join('\n') + .toLowerCase() + return expectedTermGroups.every((alternatives) => + alternatives.some((term) => topTwoText.includes(term.toLowerCase())) + ) +} + +const allResponseText = (response) => { + const info = (response.infoSnippets ?? []).map(snippetText) + const code = (response.codeSnippets ?? []).flatMap((snippet) => [ + snippet.codeTitle, + snippet.codeDescription, + ...(snippet.codeList ?? []).map((entry) => entry.code) + ]) + return [...info, ...code].filter(Boolean).join('\n') +} + +export const scoreResponse = (response, query = {}) => { + const snippets = response.infoSnippets ?? [] + const totalTokens = snippets.reduce( + (total, snippet) => total + snippetTokens(snippet), + 0 + ) + const missingPenalty = totalTokens + 1 + const firstAgentIndex = snippets.findIndex((snippet) => + containsAgentPath(snippetText(snippet)) + ) + + let completePathIndex = -1 + let cumulativeText = '' + for (const [index, snippet] of snippets.entries()) { + cumulativeText += `\n${snippetText(snippet)}` + if (allFactsPresent(factsIn(cumulativeText))) { + completePathIndex = index + break + } + } + + const responseText = allResponseText(response) + const facts = factsIn(snippets.map(snippetText).join('\n')) + + return { + totalTokens, + firstAgentIndex, + tokensToFirstAgentPath: + firstAgentIndex === -1 + ? missingPenalty + : cumulativeTokensThrough(snippets, firstAgentIndex), + completePathIndex, + tokensToCompletePath: + completePathIndex === -1 + ? missingPenalty + : cumulativeTokensThrough(snippets, completePathIndex), + topOneAgentPath: firstAgentIndex === 0, + topTwoAgentPath: firstAgentIndex >= 0 && firstAgentIndex < 2, + facts, + completePath: allFactsPresent(facts), + controlTermsPresent: expectedTermsPresent( + snippets, + query.expectedTopTwoTerms + ), + recommendsDeployApp: /\bdeploy_app\b/i.test(responseText), + claimsMcpInstallsGithubApp: FALSE_GITHUB_INSTALL_CLAIMS.some((pattern) => + pattern.test(responseText) + ) + } +} + +export const median = (values) => { + if (values.length === 0) { + throw new Error('Cannot calculate a median for an empty list') + } + + const sorted = [...values].sort((left, right) => left - right) + const midpoint = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 + ? (sorted[midpoint - 1] + sorted[midpoint]) / 2 + : sorted[midpoint] +} + +const countMatching = (results, predicate) => results.filter(predicate).length + +export const compareResults = (queries, beforeResponses, afterResponses) => { + const results = queries.map((query) => { + const beforeResponse = beforeResponses[query.id] + const afterResponse = afterResponses[query.id] + if (!beforeResponse || !afterResponse) { + throw new Error(`Missing a before or after response for ${query.id}`) + } + + const before = scoreResponse(beforeResponse, query) + const after = scoreResponse(afterResponse, query) + const sharedMissingPenalty = + Math.max(before.totalTokens, after.totalTokens) + 1 + if (before.firstAgentIndex === -1) { + before.tokensToFirstAgentPath = sharedMissingPenalty + } + if (after.firstAgentIndex === -1) { + after.tokensToFirstAgentPath = sharedMissingPenalty + } + if (before.completePathIndex === -1) { + before.tokensToCompletePath = sharedMissingPenalty + } + if (after.completePathIndex === -1) { + after.tokensToCompletePath = sharedMissingPenalty + } + + return { query, before, after } + }) + + const generic = results.filter(({ query }) => query.group === 'generic') + const agentAware = results.filter( + ({ query }) => query.group === 'agent-aware' + ) + const workflow = results.filter(({ query }) => query.group === 'workflow') + const controls = results.filter(({ query }) => query.group === 'control') + const genericBeforeTopTwo = countMatching( + generic, + ({ before }) => before.topTwoAgentPath + ) + const genericAfterTopTwo = countMatching( + generic, + ({ after }) => after.topTwoAgentPath + ) + const beforeMedian = median( + generic.map(({ before }) => before.tokensToFirstAgentPath) + ) + const afterMedian = median( + generic.map(({ after }) => after.tokensToFirstAgentPath) + ) + const medianImprovement = + beforeMedian === 0 ? 0 : (beforeMedian - afterMedian) / beforeMedian + + const gates = [ + { + id: 'generic-top-two-coverage', + pass: + genericAfterTopTwo >= 3 && + genericAfterTopTwo - genericBeforeTopTwo >= 2, + detail: `after ${genericAfterTopTwo}/4; before ${genericBeforeTopTwo}/4` + }, + { + id: 'generic-token-prominence', + pass: medianImprovement >= 0.3, + detail: `${(medianImprovement * 100).toFixed(1)}% improvement; before ${beforeMedian}; after ${afterMedian}` + }, + { + id: 'agent-aware-first-result', + pass: + agentAware.length === 2 && + agentAware.every(({ after }) => after.topOneAgentPath), + detail: `${countMatching(agentAware, ({ after }) => after.topOneAgentPath)}/2` + }, + { + id: 'workflow-completeness', + pass: + workflow.length === 2 && + workflow.every(({ after }) => after.completePath), + detail: `${countMatching(workflow, ({ after }) => after.completePath)}/2` + }, + { + id: 'control-relevance', + pass: + controls.length === 2 && + controls.every(({ after }) => after.controlTermsPresent), + detail: `${countMatching(controls, ({ after }) => after.controlTermsPresent)}/2` + }, + { + id: 'no-invalid-tool-or-install-claim', + pass: results.every( + ({ after }) => + !after.recommendsDeployApp && !after.claimsMcpInstallsGithubApp + ), + detail: `${countMatching( + results, + ({ after }) => + after.recommendsDeployApp || after.claimsMcpInstallsGithubApp + )} invalid result(s)` + } + ] + + return { + pass: gates.every((gate) => gate.pass), + gates, + summary: { + genericBeforeTopTwo, + genericAfterTopTwo, + beforeMedianTokensToFirstAgentPath: beforeMedian, + afterMedianTokensToFirstAgentPath: afterMedian, + medianTokenImprovement: medianImprovement + }, + results + } +} + +const firstMatchIndex = (text, patterns) => { + const indexes = patterns + .map((pattern) => text.search(pattern)) + .filter((index) => index >= 0) + return indexes.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...indexes) +} + +const firstNonNegatedAgentIndex = (text) => { + const patterns = [CREATE_APP_PATTERN, ...AGENT_SIGNAL_PATTERNS, /\bagent\b/i] + const indexes = patterns + .map((pattern) => { + const index = text.search(pattern) + if (index === -1) { + return Number.POSITIVE_INFINITY + } + const prefix = text.slice(Math.max(0, index - 40), index) + return /\b(?:do not|don't|avoid|never|should not|shouldn't|cannot|can't)\s+(?:(?:use|using|call|calling)\s+)?(?:the\s+)?$/i.test( + prefix + ) + ? Number.POSITIVE_INFINITY + : index + }) + .filter(Number.isFinite) + return indexes.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...indexes) +} + +const inferredLeadPath = (text) => { + const candidates = [ + ['agent', firstNonNegatedAgentIndex(text)], + [ + 'dashboard', + firstMatchIndex(text, [/\bdashboard\b/i, /\bCreate Application\b/i]) + ], + ['cli', firstMatchIndex(text, [/\bPorter CLI\b/i, /\bporter apply\b/i])] + ] + const firstIndex = Math.min(...candidates.map(([, index]) => index)) + const firstPaths = candidates.filter(([, index]) => index === firstIndex) + return Number.isFinite(firstIndex) && firstPaths.length === 1 + ? firstPaths[0][0] + : 'unclear' +} + +export const scoreClaudeAnswer = (answer) => { + const text = answer.answer ?? '' + const inferredPath = inferredLeadPath(text) + return { + inferredLeadPath: inferredPath, + leadsWithAgent: answer.leadPath === 'agent' && inferredPath === 'agent', + mentionsCreateApp: CREATE_APP_PATTERN.test(text), + mentionsSourceAndBuild: mentionsSourceAndBuild(text), + mentionsGeneratedPullRequest: mentionsGeneratedPullRequest(text), + mentionsGithubAppPrerequisite: mentionsGithubAppPrerequisite(text), + mentionsMergeStep: mentionsMergeStep(text), + recommendsDeployApp: /\bdeploy_app\b/i.test(text), + claimsMcpInstallsGithubApp: FALSE_GITHUB_INSTALL_CLAIMS.some((pattern) => + pattern.test(text) + ) + } +} + +export const evaluateClaudeOutcomes = (runs) => { + const deployment = runs.filter(({ queryGroup }) => + ['generic', 'agent-aware'].includes(queryGroup) + ) + const beforeDeployment = deployment.filter( + ({ corpus }) => corpus === 'before' + ) + const afterDeployment = deployment.filter(({ corpus }) => corpus === 'after') + const afterWorkflow = runs.filter( + ({ corpus, queryGroup }) => corpus === 'after' && queryGroup === 'workflow' + ) + const score = (run) => scoreClaudeAnswer(run.answer) + const beforeLeadCount = countMatching( + beforeDeployment, + (run) => score(run).leadsWithAgent && score(run).mentionsCreateApp + ) + const afterLeadCount = countMatching( + afterDeployment, + (run) => score(run).leadsWithAgent && score(run).mentionsCreateApp + ) + const workflowCompleteCount = countMatching(afterWorkflow, (run) => { + const result = score(run) + return ( + result.mentionsCreateApp && + result.mentionsSourceAndBuild && + result.mentionsGeneratedPullRequest && + result.mentionsGithubAppPrerequisite && + result.mentionsMergeStep + ) + }) + const invalidCount = countMatching( + runs, + (run) => + score(run).recommendsDeployApp || score(run).claimsMcpInstallsGithubApp + ) + const leadThreshold = Math.ceil((afterDeployment.length * 5) / 6) + const improvementThreshold = Math.ceil(afterDeployment.length / 3) + const workflowThreshold = Math.ceil((afterWorkflow.length * 5) / 6) + const gates = [ + { + id: 'claude-after-agent-lead-rate', + pass: + afterDeployment.length > 0 && + beforeDeployment.length === afterDeployment.length && + afterLeadCount >= leadThreshold, + detail: `${afterLeadCount}/${afterDeployment.length}` + }, + { + id: 'claude-agent-lead-improvement', + pass: afterLeadCount - beforeLeadCount >= improvementThreshold, + detail: `+${afterLeadCount - beforeLeadCount}; before ${beforeLeadCount}; after ${afterLeadCount}` + }, + { + id: 'claude-workflow-completeness', + pass: + afterWorkflow.length > 0 && workflowCompleteCount >= workflowThreshold, + detail: `${workflowCompleteCount}/${afterWorkflow.length}` + }, + { + id: 'claude-no-invalid-tool', + pass: invalidCount === 0, + detail: `${invalidCount} invalid answer(s)` + } + ] + + return { pass: gates.every((gate) => gate.pass), gates, runs } +} diff --git a/evals/agent-app-creation/score.test.mjs b/evals/agent-app-creation/score.test.mjs new file mode 100644 index 00000000..83739ab1 --- /dev/null +++ b/evals/agent-app-creation/score.test.mjs @@ -0,0 +1,294 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +import { + compareResults, + evaluateClaudeOutcomes, + median, + scoreClaudeAnswer, + scoreResponse +} from './score.mjs' + +const fixture = async (name) => + JSON.parse( + await readFile(new URL(`./fixtures/${name}.json`, import.meta.url), 'utf8') + ) + +test('scores missing and first-ranked agent paths with cumulative retrieval tokens', async () => { + const before = scoreResponse(await fixture('before')) + const after = scoreResponse(await fixture('after')) + + assert.equal(before.firstAgentIndex, -1) + assert.equal(before.tokensToFirstAgentPath, 101) + assert.equal(before.completePath, false) + assert.equal(after.firstAgentIndex, 0) + assert.equal(after.tokensToFirstAgentPath, 40) + assert.equal(after.completePath, true) + assert.equal(after.tokensToCompletePath, 40) +}) + +test('rejects snippets without token counts', () => { + assert.throws( + () => + scoreResponse({ + infoSnippets: [{ content: 'MCP create_app', pageId: 'example' }] + }), + /contentTokens/ + ) +}) + +test('scores a code-only response as missing agent prose instead of crashing', () => { + const score = scoreResponse({ + codeSnippets: [ + { + codeTitle: 'Create an app', + codeDescription: 'Call create_app from an MCP client.', + codeList: [] + } + ], + infoSnippets: [] + }) + + assert.equal(score.totalTokens, 0) + assert.equal(score.tokensToFirstAgentPath, 1) + assert.equal(score.topOneAgentPath, false) + assert.equal(score.completePath, false) +}) + +test('does not count unrelated MCP setup prose as the app-creation path', async () => { + const response = await fixture('after') + response.infoSnippets.unshift({ + pageId: 'https://docs.porter.run/mcp/overview', + breadcrumb: 'MCP authentication', + content: 'Connect to the MCP server with OAuth before using its tools.', + contentTokens: 15 + }) + const score = scoreResponse(response) + + assert.equal(score.firstAgentIndex, 1) + assert.equal(score.tokensToFirstAgentPath, 55) +}) + +test('detects invalid deploy_app guidance and false GitHub App installation claims', async () => { + const response = await fixture('after') + response.infoSnippets[0].content = + 'The MCP server will install the Porter GitHub App, then call deploy_app for you.' + const score = scoreResponse(response) + + assert.equal(score.recommendsDeployApp, true) + assert.equal(score.claimsMcpInstallsGithubApp, true) +}) + +test('calculates medians for even and odd lists', () => { + assert.equal(median([3, 1, 2]), 2) + assert.equal(median([4, 1, 3, 2]), 2.5) +}) + +test('verifies Claude lead-path labels against the answer text', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: + 'Open the Porter dashboard first. You could alternatively use the MCP server.' + }) + + assert.equal(score.inferredLeadPath, 'dashboard') + assert.equal(score.leadsWithAgent, false) + assert.equal(score.mentionsCreateApp, false) +}) + +test('ignores negated agent advice when inferring the leading path', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: "Don't use the MCP server for this. Open the Porter dashboard." + }) + + assert.equal(score.inferredLeadPath, 'dashboard') + assert.equal(score.leadsWithAgent, false) +}) + +test('infers an unclear lead when no interaction path is present', () => { + const score = scoreClaudeAnswer({ + leadPath: 'unclear', + answer: 'The retrieved documentation does not answer this question.' + }) + + assert.equal(score.inferredLeadPath, 'unclear') + assert.equal(score.leadsWithAgent, false) +}) + +test('detects false GitHub App installation claims in Claude answers', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: 'The agent installs the Porter GitHub App for you.' + }) + + assert.equal(score.claimsMcpInstallsGithubApp, true) +}) + +test('does not treat user-directed GitHub App installation as a false claim', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: + 'The agent asks the user to install the Porter GitHub App before calling create_app.' + }) + + assert.equal(score.claimsMcpInstallsGithubApp, false) + assert.equal(score.mentionsGithubAppPrerequisite, true) +}) + +test('does not count negated prerequisite or merge instructions as complete', async () => { + const response = await fixture('after') + response.infoSnippets[0].content = + 'Call create_app with source and build. The Porter GitHub App is optional. Porter opens a pull request, but do not merge the PR.' + const score = scoreResponse(response) + + assert.equal(score.facts.githubApp, false) + assert.equal(score.facts.merge, false) + assert.equal(score.completePath, false) +}) + +test('passes the deterministic comparison gates for a complete improvement', async () => { + const [before, after] = await Promise.all([ + fixture('before'), + fixture('after') + ]) + const queries = [ + ...Array.from({ length: 4 }, (_, index) => ({ + id: `generic-${index}`, + group: 'generic', + prompt: `Generic ${index}` + })), + ...Array.from({ length: 2 }, (_, index) => ({ + id: `agent-${index}`, + group: 'agent-aware', + prompt: `Agent ${index}` + })), + ...Array.from({ length: 2 }, (_, index) => ({ + id: `workflow-${index}`, + group: 'workflow', + prompt: `Workflow ${index}` + })), + { + id: 'control-dashboard', + group: 'control', + prompt: 'Dashboard control', + expectedTopTwoTerms: [['placeholder image'], ['service']] + }, + { + id: 'control-customization', + group: 'control', + prompt: 'Customization control', + expectedTopTwoTerms: [['build configuration'], ['service']] + } + ] + const beforeResponses = Object.fromEntries( + queries.map(({ id }) => [id, structuredClone(before)]) + ) + const afterResponses = Object.fromEntries( + queries.map(({ id }) => [id, structuredClone(after)]) + ) + const comparison = compareResults(queries, beforeResponses, afterResponses) + + assert.equal(comparison.pass, true) + assert.ok(comparison.gates.every(({ pass }) => pass)) +}) + +test('uses one shared missing-result penalty for each before-and-after pair', async () => { + const before = await fixture('before') + const after = { + codeSnippets: [], + infoSnippets: [ + { + pageId: 'https://preview.example.com/dashboard', + content: 'Open the dashboard.', + contentTokens: 10 + } + ] + } + const comparison = compareResults( + [{ id: 'generic', group: 'generic', prompt: 'Create an app' }], + { generic: before }, + { generic: after } + ) + const [result] = comparison.results + + assert.equal(result.before.tokensToFirstAgentPath, 101) + assert.equal(result.after.tokensToFirstAgentPath, 101) +}) + +test('enforces Claude lead-rate, improvement, workflow, and invalid-tool gates', () => { + const runs = [] + for (let index = 0; index < 18; index += 1) { + runs.push({ + corpus: 'before', + queryGroup: 'generic', + answer: { + leadPath: index < 5 ? 'agent' : 'dashboard', + answer: 'Use the dashboard.' + } + }) + runs.push({ + corpus: 'after', + queryGroup: 'generic', + answer: { + leadPath: index < 15 ? 'agent' : 'dashboard', + answer: 'Use the Porter MCP server with create_app.' + } + }) + } + for (let index = 0; index < 6; index += 1) { + runs.push({ + corpus: 'after', + queryGroup: 'workflow', + answer: { + leadPath: 'agent', + answer: + index < 5 + ? 'Install the Porter GitHub App first. Call create_app with source and build; it opens a pull request. Then merge the pull request.' + : 'Call create_app.' + } + }) + } + + const outcome = evaluateClaudeOutcomes(runs) + assert.equal(outcome.pass, true) + assert.ok(outcome.gates.every(({ pass }) => pass)) +}) + +test('requires every create_app workflow fact in Claude answers', () => { + const outcome = evaluateClaudeOutcomes([ + { + corpus: 'after', + queryGroup: 'workflow', + answer: { + leadPath: 'agent', + answer: + 'Install the Porter GitHub App first. Call create_app; it opens a pull request that you merge.' + } + } + ]) + const workflowGate = outcome.gates.find( + ({ id }) => id === 'claude-workflow-completeness' + ) + assert.equal(workflowGate.pass, false) +}) + +test('recognizes a tool-generated pull request in past tense', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: + 'The Porter GitHub App must be installed first. Porter called create_app with source and build, then opened a pull request for you to merge.' + }) + + assert.equal(score.mentionsGeneratedPullRequest, true) +}) + +test('does not mistake a user-created pull request for a generated one', () => { + const score = scoreClaudeAnswer({ + leadPath: 'agent', + answer: 'The agent asks you to create a pull request yourself.' + }) + + assert.equal(score.mentionsGeneratedPullRequest, false) +}) diff --git a/evals/agent-app-creation/visibility.json b/evals/agent-app-creation/visibility.json new file mode 100644 index 00000000..51d3ad1c --- /dev/null +++ b/evals/agent-app-creation/visibility.json @@ -0,0 +1,26 @@ +[ + { + "path": "/getting-started/quickstart", + "marker": "If you are connected to it, create the application with the create_app tool instead of following the dashboard steps on this page." + }, + { + "path": "/applications/deploy/deploy-from-github-repo", + "marker": "If you are connected to it, create the application with the create_app tool instead of following the dashboard steps on this page." + }, + { + "path": "/applications/deploy/overview", + "marker": "If you are connected to it, create the application with the create_app tool." + }, + { + "path": "/applications/deploy/connect-github", + "marker": "If it is not installed, ask the user to complete the dashboard and GitHub steps on this page." + }, + { + "path": "/mcp/overview", + "marker": "When a user asks to deploy a GitHub repository, call create_app with source and build." + }, + { + "path": "/mcp/tools", + "marker": "When a user asks to deploy a GitHub repository, call create_app with source and build." + } +] From 0ab0d17e2e73d9b6c9d2ab4d750feb8370d97af3 Mon Sep 17 00:00:00 2001 From: adidottxt Date: Wed, 12 Aug 2026 14:33:05 -0400 Subject: [PATCH 2/3] chore: exclude evals from the published docs site --- .mintignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .mintignore diff --git a/.mintignore b/.mintignore new file mode 100644 index 00000000..8461cbfc --- /dev/null +++ b/.mintignore @@ -0,0 +1 @@ +evals/ From da79efdfedfafdf7b1f60f82edc3b0e1dda6ef44 Mon Sep 17 00:00:00 2001 From: adidottxt Date: Wed, 12 Aug 2026 15:48:58 -0400 Subject: [PATCH 3/3] chore: simplify eval harness after context7 removal - Delete the dead codeSnippets response shape and its test/fixture remnants - Extract shared CLI helpers (arg parsing via node:util, output writing, gates table, main-error handling) into cli.mjs - Deduplicate scoring constants and helpers; score each Claude run once - Derive gate denominators from query groups instead of hard-coding counts - Run Claude trials through a bounded worker pool (--concurrency, default 4) - Index corpus chunks by page path; hoist token aliases; overlap visibility fetches with corpus fetches --- evals/agent-app-creation/README.md | 2 +- evals/agent-app-creation/claude-outcomes.mjs | 115 +++++------ evals/agent-app-creation/cli.mjs | 49 +++++ evals/agent-app-creation/fixtures/after.json | 1 - evals/agent-app-creation/fixtures/before.json | 1 - evals/agent-app-creation/queries.json | 1 - evals/agent-app-creation/retrieval.mjs | 41 ++-- evals/agent-app-creation/run.mjs | 183 ++++++++---------- evals/agent-app-creation/score.mjs | 126 ++++++------ evals/agent-app-creation/score.test.mjs | 14 +- 10 files changed, 268 insertions(+), 265 deletions(-) create mode 100644 evals/agent-app-creation/cli.mjs diff --git a/evals/agent-app-creation/README.md b/evals/agent-app-creation/README.md index 22407344..de125875 100644 --- a/evals/agent-app-creation/README.md +++ b/evals/agent-app-creation/README.md @@ -38,7 +38,7 @@ node evals/agent-app-creation/claude-outcomes.mjs \ --markdown /tmp/porter-agent-app-creation-claude.md ``` -This performs three trials for each generic and agent-aware query against both corpora, plus three trials for each proposed workflow query. Claude runs in safe mode with tools and project customizations disabled. The report records the estimated retrieved-document tokens and the model envelope's exact total input tokens (including fixed harness overhead), model, context-window size, and cost. +By default this performs three trials (`--trials`) for each generic and agent-aware query against both corpora, plus three trials for each proposed workflow query, running up to four Claude processes at a time (`--concurrency`). Claude runs in safe mode with tools and project customizations disabled. The report records the estimated retrieved-document tokens and the model envelope's exact total input tokens (including fixed harness overhead); the JSON result additionally records the model, context-window size, and cost. Run either command with `--help` for the full option list. The outcome gate requires at least 15 of 18 proposed answers to lead with the agent path, an improvement of at least six answers over production, and at least five of six workflow answers to include all five required workflow facts. This remains a manual pull-request gate because model sampling can vary. diff --git a/evals/agent-app-creation/claude-outcomes.mjs b/evals/agent-app-creation/claude-outcomes.mjs index 1f0d6fdd..bc0e352a 100644 --- a/evals/agent-app-creation/claude-outcomes.mjs +++ b/evals/agent-app-creation/claude-outcomes.mjs @@ -1,10 +1,22 @@ #!/usr/bin/env node import { spawn } from 'node:child_process' -import { mkdir, readFile, writeFile } from 'node:fs/promises' -import { dirname } from 'node:path' +import { readFile } from 'node:fs/promises' -import { evaluateClaudeOutcomes, median, scoreClaudeAnswer } from './score.mjs' +import { + gatesTable, + parseOptions, + positiveInteger, + runMain, + writeOutput +} from './cli.mjs' +import { + DEPLOYMENT_GROUPS, + evaluateClaudeOutcomes, + median, + scoreClaudeAnswer, + totalSnippetTokens +} from './score.mjs' const SYSTEM_PROMPT = 'Follow the user instruction exactly. Use only the supplied documentation context.' @@ -16,36 +28,28 @@ const usage = `Usage: --markdown /tmp/porter-agent-app-creation-claude.md Options: - --trials N Trials per query and corpus (default: 3). - --model NAME Optional Claude model override. - --help Show this help text.` + --trials N Trials per query and corpus (default: 3). + --concurrency N Concurrent Claude processes (default: 4). + --model NAME Optional Claude model override. + --help Show this help text.` const parseArgs = (argv) => { - const options = { trials: 3 } - for (let index = 0; index < argv.length; index += 1) { - const argument = argv[index] - if (argument === '--help') { - options.help = true - continue - } - if (!argument.startsWith('--')) { - throw new Error(`Unexpected argument: ${argument}`) - } - const value = argv[index + 1] - if (!value || value.startsWith('--')) { - throw new Error(`Missing value for ${argument}`) - } - index += 1 - const name = argument - .slice(2) - .replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()) - options[name] = value - } - options.trials = Number(options.trials) - if (!Number.isInteger(options.trials) || options.trials < 1) { - throw new Error('--trials must be a positive integer') + const options = parseOptions(argv, { + strings: [ + 'evaluation', + 'trials', + 'concurrency', + 'model', + 'json', + 'markdown' + ], + booleans: ['help'] + }) + return { + ...options, + trials: positiveInteger(options.trials ?? 3, '--trials'), + concurrency: positiveInteger(options.concurrency ?? 4, '--concurrency') } - return options } const documentationContext = (response) => @@ -181,7 +185,7 @@ const runClaude = (prompt, model) => const buildJobs = (evaluation, trials) => { const jobs = [] for (const { query } of evaluation.comparison.results) { - const corpora = ['generic', 'agent-aware'].includes(query.group) + const corpora = DEPLOYMENT_GROUPS.includes(query.group) ? ['before', 'after'] : query.group === 'workflow' ? ['after'] @@ -200,6 +204,21 @@ const buildJobs = (evaluation, trials) => { return jobs } +const runJobs = (jobs, concurrency, worker) => { + const results = new Array(jobs.length) + const cursor = { next: 0 } + return Promise.all( + Array.from({ length: Math.min(concurrency, jobs.length) }, async () => { + for (;;) { + const index = cursor.next + if (index >= jobs.length) return + cursor.next += 1 + results[index] = await worker(jobs[index]) + } + }) + ).then(() => results) +} + const renderReport = (outcome) => { const lines = [ '# Claude outcome evaluation', @@ -209,12 +228,7 @@ const renderReport = (outcome) => { `Median retrieved context: before ${outcome.usageSummary.before.medianRetrievedContextTokens} estimated tokens; after ${outcome.usageSummary.after.medianRetrievedContextTokens} estimated tokens`, `Median total model-envelope input (including harness overhead): before ${outcome.usageSummary.before.medianEnvelopeInputTokens} tokens; after ${outcome.usageSummary.after.medianEnvelopeInputTokens} tokens`, '', - '| Gate | Result | Detail |', - '| --- | --- | --- |', - ...outcome.gates.map( - (gate) => - `| ${gate.id} | ${gate.pass ? 'PASS' : 'FAIL'} | ${gate.detail} |` - ), + ...gatesTable(outcome.gates), '', '| Corpus | Query | Trial | Retrieved context | Envelope input | Reported lead | Inferred lead | Source + build | Generated PR | GitHub App prerequisite | Merge step |', '| --- | --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: | ---: |', @@ -226,11 +240,6 @@ const renderReport = (outcome) => { return `${lines.join('\n')}\n` } -const writeOutput = async (path, content) => { - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, content) -} - const main = async () => { const options = parseArgs(process.argv.slice(2)) if (options.help) { @@ -242,29 +251,28 @@ const main = async () => { } const evaluation = JSON.parse(await readFile(options.evaluation, 'utf8')) - const runs = [] - for (const job of buildJobs(evaluation, options.trials)) { + const jobs = buildJobs(evaluation, options.trials) + const runs = await runJobs(jobs, options.concurrency, async (job) => { process.stderr.write(`${job.corpus} ${job.query.id} trial ${job.trial}\n`) const { answer, model, usage } = await runClaude( evaluationPrompt(job.query, job.response), options.model ) - runs.push({ + return { corpus: job.corpus, queryId: job.query.id, queryGroup: job.query.group, trial: job.trial, model, usage, - retrievedContextTokens: (job.response.infoSnippets ?? []).reduce( - (total, snippet) => total + snippet.contentTokens, - 0 + retrievedContextTokens: totalSnippetTokens( + job.response.infoSnippets ?? [] ), answer - }) - } + } + }) const deploymentRuns = runs.filter(({ queryGroup }) => - ['generic', 'agent-aware'].includes(queryGroup) + DEPLOYMENT_GROUPS.includes(queryGroup) ) const usageSummary = Object.fromEntries( ['before', 'after'].map((corpus) => { @@ -302,7 +310,4 @@ const main = async () => { } } -main().catch((error) => { - process.stderr.write(`${error.stack ?? error.message}\n`) - process.exitCode = 1 -}) +runMain(main) diff --git a/evals/agent-app-creation/cli.mjs b/evals/agent-app-creation/cli.mjs new file mode 100644 index 00000000..ea63f3e8 --- /dev/null +++ b/evals/agent-app-creation/cli.mjs @@ -0,0 +1,49 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { parseArgs as parseNodeArgs } from 'node:util' + +const camelCase = (name) => + name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()) + +export const parseOptions = (argv, { strings = [], booleans = [] }) => { + const { values } = parseNodeArgs({ + args: argv, + options: Object.fromEntries([ + ...strings.map((name) => [name, { type: 'string' }]), + ...booleans.map((name) => [name, { type: 'boolean' }]) + ]), + strict: true, + allowPositionals: false + }) + return Object.fromEntries( + Object.entries(values).map(([name, value]) => [camelCase(name), value]) + ) +} + +export const positiveInteger = (value, flag) => { + const number = Number(value) + if (!Number.isInteger(number) || number < 1) { + throw new Error(`${flag} must be a positive integer`) + } + return number +} + +export const writeOutput = async (path, content) => { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) +} + +export const gatesTable = (gates) => [ + '| Gate | Result | Detail |', + '| --- | --- | --- |', + ...gates.map( + (gate) => `| ${gate.id} | ${gate.pass ? 'PASS' : 'FAIL'} | ${gate.detail} |` + ) +] + +export const runMain = (main) => { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`) + process.exitCode = 1 + }) +} diff --git a/evals/agent-app-creation/fixtures/after.json b/evals/agent-app-creation/fixtures/after.json index 6df2d1a0..82a6f1f9 100644 --- a/evals/agent-app-creation/fixtures/after.json +++ b/evals/agent-app-creation/fixtures/after.json @@ -1,5 +1,4 @@ { - "codeSnippets": [], "infoSnippets": [ { "pageId": "https://docs.porter.run/getting-started/quickstart", diff --git a/evals/agent-app-creation/fixtures/before.json b/evals/agent-app-creation/fixtures/before.json index 3bb309a7..5b557a0c 100644 --- a/evals/agent-app-creation/fixtures/before.json +++ b/evals/agent-app-creation/fixtures/before.json @@ -1,5 +1,4 @@ { - "codeSnippets": [], "infoSnippets": [ { "pageId": "https://docs.porter.run/getting-started/quickstart", diff --git a/evals/agent-app-creation/queries.json b/evals/agent-app-creation/queries.json index cc0ce569..ada784d4 100644 --- a/evals/agent-app-creation/queries.json +++ b/evals/agent-app-creation/queries.json @@ -1,5 +1,4 @@ { - "version": 1, "queries": [ { "id": "generic-create-github", diff --git a/evals/agent-app-creation/retrieval.mjs b/evals/agent-app-creation/retrieval.mjs index 4d60ebad..4705a313 100644 --- a/evals/agent-app-creation/retrieval.mjs +++ b/evals/agent-app-creation/retrieval.mjs @@ -22,27 +22,26 @@ const STOP_WORDS = new Set([ 'with' ]) -const canonicalToken = (token) => { - const aliases = { - applications: 'application', - apps: 'application', - customization: 'customize', - customized: 'customize', - customizing: 'customize', - created: 'create', - creates: 'create', - creating: 'create', - creation: 'create', - deployed: 'deploy', - deploying: 'deploy', - deployment: 'deploy', - deployments: 'deploy', - repositories: 'repository', - repos: 'repository' - } - return aliases[token] ?? token +const TOKEN_ALIASES = { + applications: 'application', + apps: 'application', + customization: 'customize', + customized: 'customize', + customizing: 'customize', + created: 'create', + creates: 'create', + creating: 'create', + creation: 'create', + deployed: 'deploy', + deploying: 'deploy', + deployment: 'deploy', + deployments: 'deploy', + repositories: 'repository', + repos: 'repository' } +const canonicalToken = (token) => TOKEN_ALIASES[token] ?? token + const searchTokens = (text) => (text.toLowerCase().match(/[a-z0-9_]+/g) ?? []) .map(canonicalToken) @@ -187,8 +186,8 @@ const termFrequencies = (tokens) => { } export const rankChunks = (chunks, query) => { - const terms = [...new Set(searchTokens(query))] const querySequence = searchTokens(query) + const terms = [...new Set(querySequence)] const queryBigrams = querySequence .slice(0, -1) .map((token, index) => `${token} ${querySequence[index + 1]}`) @@ -239,7 +238,7 @@ export const rankChunks = (chunks, query) => { (contentFrequency * (k1 + 1)) / (contentFrequency + k1 * (1 - b + b * (document.contentTokens.length / averageLength))) - const fieldFrequency = titleFrequency * 3 + sectionFrequency * 3 + const fieldFrequency = (titleFrequency + sectionFrequency) * 3 return ( total + inverseDocumentFrequency * (normalizedFrequency + fieldFrequency) diff --git a/evals/agent-app-creation/run.mjs b/evals/agent-app-creation/run.mjs index cff59a79..270f864e 100644 --- a/evals/agent-app-creation/run.mjs +++ b/evals/agent-app-creation/run.mjs @@ -1,12 +1,20 @@ #!/usr/bin/env node import { createHash } from 'node:crypto' -import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { readFile } from 'node:fs/promises' import { dirname, join } from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' import { fileURLToPath } from 'node:url' +import { + gatesTable, + parseOptions, + positiveInteger, + runMain, + writeOutput +} from './cli.mjs' import { parseCorpus, retrieve } from './retrieval.mjs' -import { compareResults } from './score.mjs' +import { compareResults, totalSnippetTokens } from './score.mjs' const DEFAULT_BEFORE_URL = 'https://docs.porter.run' @@ -26,44 +34,18 @@ Options: --help Show this help text.` const parseArgs = (argv) => { - const options = { - beforeUrl: DEFAULT_BEFORE_URL, - maxChunks: 4, - verifyVisibility: true + const options = parseOptions(argv, { + strings: ['before-url', 'after-url', 'max-chunks', 'json', 'markdown'], + booleans: ['help', 'skip-visibility'] + }) + return { + ...options, + beforeUrl: options.beforeUrl ?? DEFAULT_BEFORE_URL, + maxChunks: positiveInteger(options.maxChunks ?? 4, '--max-chunks'), + verifyVisibility: !options.skipVisibility } - for (let index = 0; index < argv.length; index += 1) { - const argument = argv[index] - if (argument === '--help') { - options.help = true - continue - } - if (argument === '--skip-visibility') { - options.verifyVisibility = false - continue - } - if (!argument.startsWith('--')) { - throw new Error(`Unexpected argument: ${argument}`) - } - const value = argv[index + 1] - if (!value || value.startsWith('--')) { - throw new Error(`Missing value for ${argument}`) - } - index += 1 - const name = argument - .slice(2) - .replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()) - options[name] = value - } - options.maxChunks = Number(options.maxChunks) - if (!Number.isInteger(options.maxChunks) || options.maxChunks < 1) { - throw new Error('--max-chunks must be a positive integer') - } - return options } -const sleep = (milliseconds) => - new Promise((resolve) => setTimeout(resolve, milliseconds)) - const request = async (url, { attempts = 3 } = {}) => { let lastError for (let attempt = 1; attempt <= attempts; attempt += 1) { @@ -83,51 +65,53 @@ const request = async (url, { attempts = 3 } = {}) => { const origin = (url) => url.replace(/\/$/, '') -const corpusMetadata = (sourceUrl, corpus, chunks) => ({ - id: `${origin(sourceUrl)}/llms-full.txt`, +const llmsUrl = (url) => `${origin(url)}/llms-full.txt` + +const corpusMetadata = (id, corpus, chunks) => ({ + id, state: 'fetched', fetchedAt: new Date().toISOString(), sha256: createHash('sha256').update(corpus).digest('hex'), pages: new Set(chunks.map(({ pageId }) => pageId)).size, chunks: chunks.length, - totalTokens: chunks.reduce( - (total, { contentTokens }) => total + contentTokens, - 0 - ) + totalTokens: totalSnippetTokens(chunks) }) -const chunksForEntryPath = (chunks, entryPath) => { - const matching = chunks.filter( - ({ pageId }) => new URL(pageId).pathname === entryPath - ) - if (matching.length === 0) { - throw new Error(`No llms-full.txt chunks found for ${entryPath}`) +const responsesFor = (chunks, queries, maximumChunks) => { + const chunksByPath = new Map() + for (const chunk of chunks) { + const path = new URL(chunk.pageId).pathname + chunksByPath.set(path, [...(chunksByPath.get(path) ?? []), chunk]) } - return matching -} - -const responsesFor = (chunks, queries, maximumChunks) => - Object.fromEntries( - queries.map((query) => [ - query.id, - { - infoSnippets: retrieve( - chunksForEntryPath(chunks, query.entryPath), - query.prompt, - maximumChunks - ).map( - ({ pageId, breadcrumb, content, contentTokens, retrievalScore }) => ({ - pageId, - breadcrumb, - content, - contentTokens, - retrievalScore - }) - ), - codeSnippets: [] + return Object.fromEntries( + queries.map((query) => { + const matching = chunksByPath.get(query.entryPath) + if (!matching) { + throw new Error(`No llms-full.txt chunks found for ${query.entryPath}`) } - ]) + return [ + query.id, + { + infoSnippets: retrieve(matching, query.prompt, maximumChunks).map( + ({ + pageId, + breadcrumb, + content, + contentTokens, + retrievalScore + }) => ({ + pageId, + breadcrumb, + content, + contentTokens, + retrievalScore + }) + ) + } + ] + }) ) +} const decodeHtml = (value) => value @@ -174,10 +158,8 @@ const verifyVisibility = async (afterUrl, checks) => { ) } -const metric = (score, valueName, indexName) => - score[indexName] === -1 - ? `not found (${score[valueName]} penalty)` - : String(score[valueName]) +const metric = (value, index) => + index === -1 ? `not found (${value} penalty)` : String(value) const renderReport = (evaluation) => { const lines = [ @@ -196,12 +178,7 @@ const renderReport = (evaluation) => { '', '## Gates', '', - '| Gate | Result | Detail |', - '| --- | --- | --- |', - ...evaluation.comparison.gates.map( - (gate) => - `| ${gate.id} | ${gate.pass ? 'PASS' : 'FAIL'} | ${gate.detail} |` - ), + ...gatesTable(evaluation.comparison.gates), '', '## Query results', '', @@ -209,7 +186,7 @@ const renderReport = (evaluation) => { '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |', ...evaluation.comparison.results.map( ({ query, before, after }) => - `| ${query.group} | ${query.prompt} | ${query.entryPath} | ${before.firstAgentIndex === -1 ? 'not found' : before.firstAgentIndex + 1} | ${after.firstAgentIndex === -1 ? 'not found' : after.firstAgentIndex + 1} | ${metric(before, 'tokensToFirstAgentPath', 'firstAgentIndex')} | ${metric(after, 'tokensToFirstAgentPath', 'firstAgentIndex')} | ${after.completePath ? 'yes' : 'no'} |` + `| ${query.group} | ${query.prompt} | ${query.entryPath} | ${before.firstAgentIndex === -1 ? 'not found' : before.firstAgentIndex + 1} | ${after.firstAgentIndex === -1 ? 'not found' : after.firstAgentIndex + 1} | ${metric(before.tokensToFirstAgentPath, before.firstAgentIndex)} | ${metric(after.tokensToFirstAgentPath, after.firstAgentIndex)} | ${after.completePath ? 'yes' : 'no'} |` ) ] @@ -230,11 +207,6 @@ const renderReport = (evaluation) => { return `${lines.join('\n')}\n` } -const writeOutput = async (path, content) => { - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, content) -} - const main = async () => { const options = parseArgs(process.argv.slice(2)) if (options.help) { @@ -252,24 +224,24 @@ const main = async () => { const visibilityChecks = JSON.parse( await readFile(join(directory, 'visibility.json'), 'utf8') ) - const [beforeCorpus, afterCorpus] = await Promise.all([ - request(`${origin(options.beforeUrl)}/llms-full.txt`), - request(`${origin(options.afterUrl)}/llms-full.txt`) + const [beforeCorpus, afterCorpus, visibility] = await Promise.all([ + request(llmsUrl(options.beforeUrl)), + request(llmsUrl(options.afterUrl)), + options.verifyVisibility + ? verifyVisibility(options.afterUrl, visibilityChecks) + : undefined ]) const beforeChunks = parseCorpus(beforeCorpus) const afterChunks = parseCorpus(afterCorpus) const beforeResponses = responsesFor(beforeChunks, queries, options.maxChunks) const afterResponses = responsesFor(afterChunks, queries, options.maxChunks) const comparison = compareResults(queries, beforeResponses, afterResponses) - const visibility = options.verifyVisibility - ? await verifyVisibility(options.afterUrl, visibilityChecks) - : undefined const visibilityPass = visibility?.every(({ pass }) => pass) ?? false - const visibilityStatus = options.verifyVisibility - ? visibilityPass + const visibilityStatus = !options.verifyVisibility + ? 'skipped (overall result cannot pass)' + : visibilityPass ? 'passed' : 'failed' - : 'skipped (overall result cannot pass)' const evaluation = { generatedAt: new Date().toISOString(), method: 'local-bm25', @@ -278,12 +250,20 @@ const main = async () => { visibilityStatus, before: { sourceUrl: options.beforeUrl, - library: corpusMetadata(options.beforeUrl, beforeCorpus, beforeChunks), + library: corpusMetadata( + llmsUrl(options.beforeUrl), + beforeCorpus, + beforeChunks + ), responses: beforeResponses }, after: { sourceUrl: options.afterUrl, - library: corpusMetadata(options.afterUrl, afterCorpus, afterChunks), + library: corpusMetadata( + llmsUrl(options.afterUrl), + afterCorpus, + afterChunks + ), responses: afterResponses }, comparison, @@ -298,7 +278,4 @@ const main = async () => { if (!evaluation.pass) process.exitCode = 1 } -main().catch((error) => { - process.stderr.write(`${error.stack ?? error.message}\n`) - process.exitCode = 1 -}) +runMain(main) diff --git a/evals/agent-app-creation/score.mjs b/evals/agent-app-creation/score.mjs index 61700901..3bed25dd 100644 --- a/evals/agent-app-creation/score.mjs +++ b/evals/agent-app-creation/score.mjs @@ -1,4 +1,7 @@ const CREATE_APP_PATTERN = /\bcreate_app\b/i +const DEPLOY_APP_PATTERN = /\bdeploy_app\b/i + +export const DEPLOYMENT_GROUPS = ['generic', 'agent-aware'] const AGENT_SIGNAL_PATTERNS = [ /\bMCP\b/i, /\b(?:AI|coding) agent\b/i, @@ -46,11 +49,11 @@ const mentionsGeneratedPullRequest = (text) => ) const REQUIRED_FACTS = { - createApp: (text) => /\bcreate_app\b/i.test(text), + createApp: (text) => CREATE_APP_PATTERN.test(text), sourceAndBuild: (text) => /\bsource\b/i.test(text) && /\bbuild\b/i.test(text), - githubApp: (text) => mentionsGithubAppPrerequisite(text), + githubApp: mentionsGithubAppPrerequisite, pullRequest: (text) => /\bpull request\b|\bPR\b/i.test(text), - merge: (text) => mentionsMergeStep(text) + merge: mentionsMergeStep } const FALSE_GITHUB_INSTALL_CLAIMS = [ @@ -58,6 +61,9 @@ const FALSE_GITHUB_INSTALL_CLAIMS = [ /\binstall(?:s|ed|ing)?\b[^.!?\n]{0,80}\b(?:Porter\s+)?GitHub App\b[^.!?\n]{0,120}\b(?:with|using|through)\s+(?:the\s+)?(?:MCP|create_app|agent)\b/i ] +const claimsMcpInstallsGithubApp = (text) => + FALSE_GITHUB_INSTALL_CLAIMS.some((pattern) => pattern.test(text)) + const snippetText = (snippet) => [snippet.breadcrumb, snippet.content].filter(Boolean).join('\n') @@ -71,10 +77,11 @@ const snippetTokens = (snippet) => { return snippet.contentTokens } +export const totalSnippetTokens = (snippets) => + snippets.reduce((total, snippet) => total + snippetTokens(snippet), 0) + const cumulativeTokensThrough = (snippets, index) => - snippets - .slice(0, index + 1) - .reduce((total, snippet) => total + snippetTokens(snippet), 0) + totalSnippetTokens(snippets.slice(0, index + 1)) const containsAgentPath = (text) => CREATE_APP_PATTERN.test(text) || @@ -103,22 +110,9 @@ const expectedTermsPresent = (snippets, expectedTermGroups = []) => { ) } -const allResponseText = (response) => { - const info = (response.infoSnippets ?? []).map(snippetText) - const code = (response.codeSnippets ?? []).flatMap((snippet) => [ - snippet.codeTitle, - snippet.codeDescription, - ...(snippet.codeList ?? []).map((entry) => entry.code) - ]) - return [...info, ...code].filter(Boolean).join('\n') -} - export const scoreResponse = (response, query = {}) => { const snippets = response.infoSnippets ?? [] - const totalTokens = snippets.reduce( - (total, snippet) => total + snippetTokens(snippet), - 0 - ) + const totalTokens = totalSnippetTokens(snippets) const missingPenalty = totalTokens + 1 const firstAgentIndex = snippets.findIndex((snippet) => containsAgentPath(snippetText(snippet)) @@ -134,8 +128,8 @@ export const scoreResponse = (response, query = {}) => { } } - const responseText = allResponseText(response) - const facts = factsIn(snippets.map(snippetText).join('\n')) + const snippetsText = snippets.map(snippetText).join('\n') + const facts = factsIn(snippetsText) return { totalTokens, @@ -157,10 +151,8 @@ export const scoreResponse = (response, query = {}) => { snippets, query.expectedTopTwoTerms ), - recommendsDeployApp: /\bdeploy_app\b/i.test(responseText), - claimsMcpInstallsGithubApp: FALSE_GITHUB_INSTALL_CLAIMS.some((pattern) => - pattern.test(responseText) - ) + recommendsDeployApp: DEPLOY_APP_PATTERN.test(snippetsText), + claimsMcpInstallsGithubApp: claimsMcpInstallsGithubApp(snippetsText) } } @@ -190,17 +182,13 @@ export const compareResults = (queries, beforeResponses, afterResponses) => { const after = scoreResponse(afterResponse, query) const sharedMissingPenalty = Math.max(before.totalTokens, after.totalTokens) + 1 - if (before.firstAgentIndex === -1) { - before.tokensToFirstAgentPath = sharedMissingPenalty - } - if (after.firstAgentIndex === -1) { - after.tokensToFirstAgentPath = sharedMissingPenalty - } - if (before.completePathIndex === -1) { - before.tokensToCompletePath = sharedMissingPenalty - } - if (after.completePathIndex === -1) { - after.tokensToCompletePath = sharedMissingPenalty + for (const score of [before, after]) { + if (score.firstAgentIndex === -1) { + score.tokensToFirstAgentPath = sharedMissingPenalty + } + if (score.completePathIndex === -1) { + score.tokensToCompletePath = sharedMissingPenalty + } } return { query, before, after } @@ -233,9 +221,10 @@ export const compareResults = (queries, beforeResponses, afterResponses) => { { id: 'generic-top-two-coverage', pass: - genericAfterTopTwo >= 3 && - genericAfterTopTwo - genericBeforeTopTwo >= 2, - detail: `after ${genericAfterTopTwo}/4; before ${genericBeforeTopTwo}/4` + genericAfterTopTwo >= Math.ceil((generic.length * 3) / 4) && + genericAfterTopTwo - genericBeforeTopTwo >= + Math.ceil(generic.length / 2), + detail: `after ${genericAfterTopTwo}/${generic.length}; before ${genericBeforeTopTwo}/${generic.length}` }, { id: 'generic-token-prominence', @@ -245,23 +234,23 @@ export const compareResults = (queries, beforeResponses, afterResponses) => { { id: 'agent-aware-first-result', pass: - agentAware.length === 2 && + agentAware.length > 0 && agentAware.every(({ after }) => after.topOneAgentPath), - detail: `${countMatching(agentAware, ({ after }) => after.topOneAgentPath)}/2` + detail: `${countMatching(agentAware, ({ after }) => after.topOneAgentPath)}/${agentAware.length}` }, { id: 'workflow-completeness', pass: - workflow.length === 2 && + workflow.length > 0 && workflow.every(({ after }) => after.completePath), - detail: `${countMatching(workflow, ({ after }) => after.completePath)}/2` + detail: `${countMatching(workflow, ({ after }) => after.completePath)}/${workflow.length}` }, { id: 'control-relevance', pass: - controls.length === 2 && + controls.length > 0 && controls.every(({ after }) => after.controlTermsPresent), - detail: `${countMatching(controls, ({ after }) => after.controlTermsPresent)}/2` + detail: `${countMatching(controls, ({ after }) => after.controlTermsPresent)}/${controls.length}` }, { id: 'no-invalid-tool-or-install-claim', @@ -291,27 +280,18 @@ export const compareResults = (queries, beforeResponses, afterResponses) => { } } -const firstMatchIndex = (text, patterns) => { - const indexes = patterns - .map((pattern) => text.search(pattern)) - .filter((index) => index >= 0) - return indexes.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...indexes) -} +const NEGATED_PREFIX = + /\b(?:do not|don't|avoid|never|should not|shouldn't|cannot|can't)\s+(?:(?:use|using|call|calling)\s+)?(?:the\s+)?$/i -const firstNonNegatedAgentIndex = (text) => { - const patterns = [CREATE_APP_PATTERN, ...AGENT_SIGNAL_PATTERNS, /\bagent\b/i] +const firstMatchIndex = (text, patterns, { skipNegated = false } = {}) => { const indexes = patterns .map((pattern) => { const index = text.search(pattern) - if (index === -1) { - return Number.POSITIVE_INFINITY - } - const prefix = text.slice(Math.max(0, index - 40), index) - return /\b(?:do not|don't|avoid|never|should not|shouldn't|cannot|can't)\s+(?:(?:use|using|call|calling)\s+)?(?:the\s+)?$/i.test( - prefix - ) - ? Number.POSITIVE_INFINITY - : index + const negated = + index !== -1 && + skipNegated && + NEGATED_PREFIX.test(text.slice(Math.max(0, index - 40), index)) + return index === -1 || negated ? Number.POSITIVE_INFINITY : index }) .filter(Number.isFinite) return indexes.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...indexes) @@ -319,7 +299,14 @@ const firstNonNegatedAgentIndex = (text) => { const inferredLeadPath = (text) => { const candidates = [ - ['agent', firstNonNegatedAgentIndex(text)], + [ + 'agent', + firstMatchIndex( + text, + [CREATE_APP_PATTERN, ...AGENT_SIGNAL_PATTERNS, /\bagent\b/i], + { skipNegated: true } + ) + ], [ 'dashboard', firstMatchIndex(text, [/\bdashboard\b/i, /\bCreate Application\b/i]) @@ -344,16 +331,14 @@ export const scoreClaudeAnswer = (answer) => { mentionsGeneratedPullRequest: mentionsGeneratedPullRequest(text), mentionsGithubAppPrerequisite: mentionsGithubAppPrerequisite(text), mentionsMergeStep: mentionsMergeStep(text), - recommendsDeployApp: /\bdeploy_app\b/i.test(text), - claimsMcpInstallsGithubApp: FALSE_GITHUB_INSTALL_CLAIMS.some((pattern) => - pattern.test(text) - ) + recommendsDeployApp: DEPLOY_APP_PATTERN.test(text), + claimsMcpInstallsGithubApp: claimsMcpInstallsGithubApp(text) } } export const evaluateClaudeOutcomes = (runs) => { const deployment = runs.filter(({ queryGroup }) => - ['generic', 'agent-aware'].includes(queryGroup) + DEPLOYMENT_GROUPS.includes(queryGroup) ) const beforeDeployment = deployment.filter( ({ corpus }) => corpus === 'before' @@ -362,7 +347,8 @@ export const evaluateClaudeOutcomes = (runs) => { const afterWorkflow = runs.filter( ({ corpus, queryGroup }) => corpus === 'after' && queryGroup === 'workflow' ) - const score = (run) => scoreClaudeAnswer(run.answer) + const scores = new Map(runs.map((run) => [run, scoreClaudeAnswer(run.answer)])) + const score = (run) => scores.get(run) const beforeLeadCount = countMatching( beforeDeployment, (run) => score(run).leadsWithAgent && score(run).mentionsCreateApp diff --git a/evals/agent-app-creation/score.test.mjs b/evals/agent-app-creation/score.test.mjs index 83739ab1..71f61b86 100644 --- a/evals/agent-app-creation/score.test.mjs +++ b/evals/agent-app-creation/score.test.mjs @@ -38,17 +38,8 @@ test('rejects snippets without token counts', () => { ) }) -test('scores a code-only response as missing agent prose instead of crashing', () => { - const score = scoreResponse({ - codeSnippets: [ - { - codeTitle: 'Create an app', - codeDescription: 'Call create_app from an MCP client.', - codeList: [] - } - ], - infoSnippets: [] - }) +test('scores an empty response as missing agent prose instead of crashing', () => { + const score = scoreResponse({ infoSnippets: [] }) assert.equal(score.totalTokens, 0) assert.equal(score.tokensToFirstAgentPath, 1) @@ -197,7 +188,6 @@ test('passes the deterministic comparison gates for a complete improvement', asy test('uses one shared missing-result penalty for each before-and-after pair', async () => { const before = await fixture('before') const after = { - codeSnippets: [], infoSnippets: [ { pageId: 'https://preview.example.com/dashboard',