chore: add agent app-creation docs evaluation harness

.mintignore

@@ -0,0 +1 @@
+evals/

evals/agent-app-creation/README.md

@@ -0,0 +1,51 @@
+# Agent app-creation documentation evaluation
+
+This evaluation measures whether Porter documentation makes the agent path prominent for application creation. The primary harness is local and reproducible: it fetches the live and proposed `llms-full.txt` files, selects each query's explicit canonical entry page, chunks both versions identically, ranks sections within that page with a small fielded BM25 implementation, and compares the context required to discover the agent workflow.
+
+The primary metric is **tokens to first agent path**: the cumulative estimated documentation tokens before a ranked chunk names `create_app`, or connects the Porter MCP server or an agent to creating or deploying an application. Missing guidance receives one shared before-and-after penalty. The estimate is stable UTF-8 bytes divided by four, so it is suitable for relative comparisons without depending on a vendor tokenizer. The test measures prominence after an agent fetches an app-creation page.
+
+The complete path requires all of the following facts:
+
+- Call `create_app`.
+- Pass `source` and `build`.
+- Install the Porter GitHub App before calling the tool.
+- Porter opens a pull request.
+- Merge the pull request to trigger the first real deployment.
+
+## Run the local retrieval evaluation
+
+The preview must be public and expose `llms-full.txt` and `.md` page exports.
+
+```bash
+node evals/agent-app-creation/run.mjs \
+ --after-url https://porter-preview.mintlify.site \
+ --json /tmp/porter-agent-app-creation.json \
+ --markdown /tmp/porter-agent-app-creation.md
+```
+
+The report records SHA-256 hashes for both input corpora. It also verifies that the agent-only direction appears in each proposed `.md` export but not in the visible HTML page.
+
+By default, each answer receives only the top four chunks, creating a small and explicit documentation-context budget while preserving the stricter top-two ranking gates. Use `--max-chunks` to sweep smaller or larger windows when diagnosing a failure.
+
+## Run the Claude outcome evaluation
+
+Use the generated retrieval JSON as the only documentation context supplied to local Claude:
+
+```bash
+node evals/agent-app-creation/claude-outcomes.mjs \
+ --evaluation /tmp/porter-agent-app-creation.json \
+ --json /tmp/porter-agent-app-creation-claude.json \
+ --markdown /tmp/porter-agent-app-creation-claude.md
+```
+
+By default this performs three trials (`--trials`) for each generic and agent-aware query against both corpora, plus three trials for each proposed workflow query, running up to four Claude processes at a time (`--concurrency`). Claude runs in safe mode with tools and project customizations disabled. The report records the estimated retrieved-document tokens and the model envelope's exact total input tokens (including fixed harness overhead); the JSON result additionally records the model, context-window size, and cost. Run either command with `--help` for the full option list.
+
+The outcome gate requires at least 15 of 18 proposed answers to lead with the agent path, an improvement of at least six answers over production, and at least five of six workflow answers to include all five required workflow facts. This remains a manual pull-request gate because model sampling can vary.
+
+## Test the scorers and retriever
+
+```bash
+node --test \
+ evals/agent-app-creation/score.test.mjs \
+ evals/agent-app-creation/retrieval.test.mjs
+```

evals/agent-app-creation/claude-outcomes.mjs

@@ -0,0 +1,313 @@
+#!/usr/bin/env node
+
+import { spawn } from 'node:child_process'
+import { readFile } from 'node:fs/promises'
+
+import {
+ gatesTable,
+ parseOptions,
+ positiveInteger,
+ runMain,
+ writeOutput
+} from './cli.mjs'
+import {
+ DEPLOYMENT_GROUPS,
+ evaluateClaudeOutcomes,
+ median,
+ scoreClaudeAnswer,
+ totalSnippetTokens
+} from './score.mjs'
+
+const SYSTEM_PROMPT =
+ 'Follow the user instruction exactly. Use only the supplied documentation context.'
+
+const usage = `Usage:
+ node evals/agent-app-creation/claude-outcomes.mjs \\
+ --evaluation /tmp/porter-agent-app-creation.json \\
+ --json /tmp/porter-agent-app-creation-claude.json \\
+ --markdown /tmp/porter-agent-app-creation-claude.md
+
+Options:
+ --trials N Trials per query and corpus (default: 3).
+ --concurrency N Concurrent Claude processes (default: 4).
+ --model NAME Optional Claude model override.
+ --help Show this help text.`
+
+const parseArgs = (argv) => {
+ const options = parseOptions(argv, {
+ strings: [
+ 'evaluation',
+ 'trials',
+ 'concurrency',
+ 'model',
+ 'json',
+ 'markdown'
+ ],
+ booleans: ['help']
+ })
+ return {
+ ...options,
+ trials: positiveInteger(options.trials ?? 3, '--trials'),
+ concurrency: positiveInteger(options.concurrency ?? 4, '--concurrency')
+ }
+}
+
+const documentationContext = (response) =>
+ (response.infoSnippets ?? [])
+ .map(
+ (snippet, index) =>
+ `Snippet ${index + 1}\nSource: ${snippet.pageId}\nSection: ${snippet.breadcrumb ?? 'Untitled'}\n${snippet.content}`
+ )
+ .join('\n\n')
+
+const evaluationPrompt = (
+ query,
+ response
+) => `You are evaluating whether retrieved Porter documentation helps an agent answer a deployment question.
+
+Answer the user using only the documentation snippets below. Do not use outside knowledge. Recommend the best available interaction path first. Do not discuss this evaluation.
+
+Return one JSON object and nothing else:
+{"leadPath":"agent|dashboard|cli|unclear","answer":"your concise answer"}
+
+Use leadPath "agent" only when the answer recommends an AI or coding agent using Porter's MCP server as the first path.
+
+User question:
+${query.prompt}
+
+Documentation snippets:
+${documentationContext(response)}`
+
+const parseClaudeResult = (stdout) => {
+ const envelope = JSON.parse(stdout)
+ const result =
+ typeof envelope.result === 'string'
+ ? envelope.result.trim()
+ : envelope.result
+ const parsed =
+ typeof result === 'object' && result
+ ? result
+ : JSON.parse(
+ String(result)
+ .replace(/^```(?:json)?\s*/i, '')
+ .replace(/\s*```$/, '')
+ )
+ if (!['agent', 'dashboard', 'cli', 'unclear'].includes(parsed.leadPath)) {
+ throw new Error(`Claude returned an invalid leadPath: ${parsed.leadPath}`)
+ }
+ if (typeof parsed.answer !== 'string' || parsed.answer.length === 0) {
+ throw new Error('Claude returned an empty answer')
+ }
+ const modelUsage = Object.entries(envelope.modelUsage ?? {})
+ const primaryModel =
+ modelUsage.find(
+ ([, usage]) => usage.outputTokens === envelope.usage?.output_tokens
+ ) ??
+ modelUsage.sort(([, left], [, right]) => right.costUSD - left.costUSD)[0]
+ const directUsage = envelope.usage ?? {}
+ const effectiveInputTokens =
+ (directUsage.input_tokens ?? 0) +
+ (directUsage.cache_creation_input_tokens ?? 0) +
+ (directUsage.cache_read_input_tokens ?? 0)
+ return {
+ answer: parsed,
+ model: primaryModel?.[0] ?? envelope.model ?? 'unknown',
+ usage: {
+ envelopeInputTokens:
+ effectiveInputTokens || primaryModel?.[1].inputTokens,
+ uncachedInputTokens: directUsage.input_tokens,
+ cacheCreationInputTokens: directUsage.cache_creation_input_tokens,
+ cacheReadInputTokens: directUsage.cache_read_input_tokens,
+ outputTokens:
+ envelope.usage?.output_tokens ?? primaryModel?.[1].outputTokens,
+ contextWindow: primaryModel?.[1].contextWindow,
+ costUSD: envelope.total_cost_usd
+ }
+ }
+}
+
+const runClaude = (prompt, model) =>
+ new Promise((resolve, reject) => {
+ const arguments_ = [
+ '-p',
+ '--output-format',
+ 'json',
+ '--no-session-persistence',
+ '--tools',
+ '',
+ '--safe-mode',
+ '--system-prompt',
+ SYSTEM_PROMPT,
+ '--disable-slash-commands'
+ ]
+ if (model) {
+ arguments_.push('--model', model)
+ }
+ const child = spawn('claude', arguments_, {
+ stdio: ['pipe', 'pipe', 'pipe']
+ })
+ let stdout = ''
+ let stderr = ''
+ const timeout = setTimeout(() => {
+ child.kill('SIGTERM')
+ reject(new Error('Claude evaluation timed out after 120 seconds'))
+ }, 120_000)
+
+ child.stdout.on('data', (chunk) => {
+ stdout += chunk
+ })
+ child.stderr.on('data', (chunk) => {
+ stderr += chunk
+ })
+ child.on('error', (error) => {
+ clearTimeout(timeout)
+ reject(error)
+ })
+ child.on('close', (code) => {
+ clearTimeout(timeout)
+ if (code !== 0) {
+ reject(new Error(`Claude exited with ${code}: ${stderr.trim()}`))
+ return
+ }
+ try {
+ resolve(parseClaudeResult(stdout))
+ } catch (error) {
+ reject(
+ new Error(
+ `Could not parse Claude output: ${error.message}\n${stdout.slice(0, 500)}`
+ )
+ )
+ }
+ })
+ child.stdin.end(prompt)
+ })
+
+const buildJobs = (evaluation, trials) => {
+ const jobs = []
+ for (const { query } of evaluation.comparison.results) {
+ const corpora = DEPLOYMENT_GROUPS.includes(query.group)
+ ? ['before', 'after']
+ : query.group === 'workflow'
+ ? ['after']
+ : []
+ for (const corpus of corpora) {
+ for (let trial = 1; trial <= trials; trial += 1) {
+ jobs.push({
+ corpus,
+ query,
+ response: evaluation[corpus].responses[query.id],
+ trial
+ })
+ }
+ }
+ }
+ return jobs
+}
+
+const runJobs = (jobs, concurrency, worker) => {
+ const results = new Array(jobs.length)
+ const cursor = { next: 0 }
+ return Promise.all(
+ Array.from({ length: Math.min(concurrency, jobs.length) }, async () => {
+ for (;;) {
+ const index = cursor.next
+ if (index >= jobs.length) return
+ cursor.next += 1
+ results[index] = await worker(jobs[index])
+ }
+ })
+ ).then(() => results)
+}
+
+const renderReport = (outcome) => {
+ const lines = [
+ '# Claude outcome evaluation',
+ '',
+ `Result: **${outcome.pass ? 'PASS' : 'FAIL'}**`,
+ `Models: ${[...new Set(outcome.runs.map((run) => run.model))].join(', ')}`,
+ `Median retrieved context: before ${outcome.usageSummary.before.medianRetrievedContextTokens} estimated tokens; after ${outcome.usageSummary.after.medianRetrievedContextTokens} estimated tokens`,
+ `Median total model-envelope input (including harness overhead): before ${outcome.usageSummary.before.medianEnvelopeInputTokens} tokens; after ${outcome.usageSummary.after.medianEnvelopeInputTokens} tokens`,
+ '',
+ ...gatesTable(outcome.gates),
+ '',
+ '| Corpus | Query | Trial | Retrieved context | Envelope input | Reported lead | Inferred lead | Source + build | Generated PR | GitHub App prerequisite | Merge step |',
+ '| --- | --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: | ---: |',
+ ...outcome.runs.map((run) => {
+ const score = scoreClaudeAnswer(run.answer)
+ return `| ${run.corpus} | ${run.queryId} | ${run.trial} | ${run.retrievedContextTokens} | ${run.usage.envelopeInputTokens} | ${run.answer.leadPath} | ${score.inferredLeadPath} | ${score.mentionsSourceAndBuild ? 'yes' : 'no'} | ${score.mentionsGeneratedPullRequest ? 'yes' : 'no'} | ${score.mentionsGithubAppPrerequisite ? 'yes' : 'no'} | ${score.mentionsMergeStep ? 'yes' : 'no'} |`
+ })
+ ]
+ return `${lines.join('\n')}\n`
+}
+
+const main = async () => {
+ const options = parseArgs(process.argv.slice(2))
+ if (options.help) {
+ process.stdout.write(`${usage}\n`)
+ return
+ }
+ if (!options.evaluation) {
+ throw new Error(`Missing required option --evaluation\n\n${usage}`)
+ }
+
+ const evaluation = JSON.parse(await readFile(options.evaluation, 'utf8'))
+ const jobs = buildJobs(evaluation, options.trials)
+ const runs = await runJobs(jobs, options.concurrency, async (job) => {
+ process.stderr.write(`${job.corpus} ${job.query.id} trial ${job.trial}\n`)
+ const { answer, model, usage } = await runClaude(
+ evaluationPrompt(job.query, job.response),
+ options.model
+ )
+ return {
+ corpus: job.corpus,
+ queryId: job.query.id,
+ queryGroup: job.query.group,
+ trial: job.trial,
+ model,
+ usage,
+ retrievedContextTokens: totalSnippetTokens(
+ job.response.infoSnippets ?? []
+ ),
+ answer
+ }
+ })
+ const deploymentRuns = runs.filter(({ queryGroup }) =>
+ DEPLOYMENT_GROUPS.includes(queryGroup)
+ )
+ const usageSummary = Object.fromEntries(
+ ['before', 'after'].map((corpus) => {
+ const corpusRuns = deploymentRuns.filter((run) => run.corpus === corpus)
+ return [
+ corpus,
+ {
+ medianRetrievedContextTokens: median(
+ corpusRuns.map(
+ ({ retrievedContextTokens }) => retrievedContextTokens
+ )
+ ),
+ medianEnvelopeInputTokens: median(
+ corpusRuns.map(({ usage }) => usage.envelopeInputTokens)
+ )
+ }
+ ]
+ })
+ )
+ const outcome = {
+ generatedAt: new Date().toISOString(),
+ usageSummary,
+ ...evaluateClaudeOutcomes(runs)
+ }
+ const report = renderReport(outcome)
+ if (options.json) {
+ await writeOutput(options.json, `${JSON.stringify(outcome, null, 2)}\n`)
+ }
+ if (options.markdown) {
+ await writeOutput(options.markdown, report)
+ }
+ process.stdout.write(report)
+ if (!outcome.pass) {
+ process.exitCode = 1
+ }
+}
+
+runMain(main)