diff --git a/.changeset/docs-drift-tooling.md b/.changeset/docs-drift-tooling.md new file mode 100644 index 0000000000..908dcb7616 --- /dev/null +++ b/.changeset/docs-drift-tooling.md @@ -0,0 +1,4 @@ +--- +--- + +chore(docs): add recurring doc-accuracy verification tooling — `affected-docs.mjs` change→docs mapping, an advisory `docs-drift-check` CI gate, and a reusable `docs-accuracy-audit` workflow. Repo-internal tooling; no package version impact. diff --git a/.claude/workflows/docs-accuracy-audit.js b/.claude/workflows/docs-accuracy-audit.js new file mode 100644 index 0000000000..eecd00d47c --- /dev/null +++ b/.claude/workflows/docs-accuracy-audit.js @@ -0,0 +1,171 @@ +export const meta = { + name: 'docs-accuracy-audit', + description: 'Audit + fix hand-written ObjectStack docs against actual implementation, with adversarial verification. Scope with args.docs; defaults to all hand-written docs.', + whenToUse: 'Periodic or change-scoped documentation accuracy verification. Pass args.docs = [paths] to scope (e.g. output of scripts/docs-audit/affected-docs.mjs); omit for a full audit of every hand-written doc.', + phases: [ + { title: 'Audit & Fix', detail: 'one agent per doc: read, locate implementation, apply evidence-backed edits' }, + { title: 'Adversarial Verify', detail: 'second agent re-checks each applied fix against code, repairs regressions' }, + ], +} + +// Default scope = every hand-written doc (content/docs/** minus references/). Keep in +// sync with `node scripts/docs-audit/affected-docs.mjs --all`. Callers normally pass a +// scoped subset via args.docs (e.g. only docs whose backing packages/ code changed). +const ALL_HANDWRITTEN = ["content/docs/concepts/architecture.mdx","content/docs/concepts/cloud-artifact-api.mdx","content/docs/concepts/cluster-semantics.mdx","content/docs/concepts/core/architecture.mdx","content/docs/concepts/core/events.mdx","content/docs/concepts/core/index.mdx","content/docs/concepts/core/plugins.mdx","content/docs/concepts/core/services.mdx","content/docs/concepts/design-principles.mdx","content/docs/concepts/implementation-status.mdx","content/docs/concepts/index.mdx","content/docs/concepts/metadata-driven.mdx","content/docs/concepts/metadata-lifecycle.mdx","content/docs/concepts/north-star.mdx","content/docs/concepts/packages.mdx","content/docs/concepts/setup-app.mdx","content/docs/concepts/skills.mdx","content/docs/concepts/terminology.mdx","content/docs/concepts/webhook-delivery.mdx","content/docs/getting-started/architecture.mdx","content/docs/getting-started/cli.mdx","content/docs/getting-started/core-concepts.mdx","content/docs/getting-started/examples.mdx","content/docs/getting-started/glossary.mdx","content/docs/getting-started/index.mdx","content/docs/getting-started/quick-start.mdx","content/docs/guides/adding-a-metadata-type.mdx","content/docs/guides/ai-capabilities.mdx","content/docs/guides/airtable-dashboard-analysis.mdx","content/docs/guides/analytics-datasets.mdx","content/docs/guides/api-reference.mdx","content/docs/guides/auth-sso.mdx","content/docs/guides/authentication.mdx","content/docs/guides/business-logic.mdx","content/docs/guides/cheatsheets/backward-compatibility.mdx","content/docs/guides/cheatsheets/error-catalog.mdx","content/docs/guides/cheatsheets/field-type-decision-tree.mdx","content/docs/guides/cheatsheets/field-type-gallery.mdx","content/docs/guides/cheatsheets/field-validation-rules.mdx","content/docs/guides/cheatsheets/permissions-matrix.mdx","content/docs/guides/cheatsheets/protocol-diagram.mdx","content/docs/guides/cheatsheets/query-cheat-sheet.mdx","content/docs/guides/cheatsheets/quick-reference.mdx","content/docs/guides/cheatsheets/wire-format.mdx","content/docs/guides/client-sdk.mdx","content/docs/guides/cloud-deployment.mdx","content/docs/guides/common-patterns.mdx","content/docs/guides/contracts/auth-service.mdx","content/docs/guides/contracts/cache-service.mdx","content/docs/guides/contracts/data-engine.mdx","content/docs/guides/contracts/index.mdx","content/docs/guides/contracts/metadata-service.mdx","content/docs/guides/contracts/storage-service.mdx","content/docs/guides/data-flow.mdx","content/docs/guides/data-modeling.mdx","content/docs/guides/deployment-vercel.mdx","content/docs/guides/driver-configuration.mdx","content/docs/guides/environment-variables.mdx","content/docs/guides/error-handling-client.mdx","content/docs/guides/error-handling-server.mdx","content/docs/guides/formula.mdx","content/docs/guides/hook-bodies.mdx","content/docs/guides/index.mdx","content/docs/guides/kernel-services.mdx","content/docs/guides/metadata/app.mdx","content/docs/guides/metadata/dashboard.mdx","content/docs/guides/metadata/doc.mdx","content/docs/guides/metadata/field.mdx","content/docs/guides/metadata/flow.mdx","content/docs/guides/metadata/index.mdx","content/docs/guides/metadata/object.mdx","content/docs/guides/metadata/page.mdx","content/docs/guides/metadata/permission.mdx","content/docs/guides/metadata/validation.mdx","content/docs/guides/metadata/view.mdx","content/docs/guides/metadata/workflow.mdx","content/docs/guides/objectql-migration.mdx","content/docs/guides/packages.mdx","content/docs/guides/plugin-chatbot-integration.mdx","content/docs/guides/plugin-development.mdx","content/docs/guides/plugins.mdx","content/docs/guides/production-readiness.mdx","content/docs/guides/project-scoping.mdx","content/docs/guides/public-forms.mdx","content/docs/guides/publish-and-preview.mdx","content/docs/guides/runtime-services/audit-service.mdx","content/docs/guides/runtime-services/data-service.mdx","content/docs/guides/runtime-services/email-service.mdx","content/docs/guides/runtime-services/examples.mdx","content/docs/guides/runtime-services/index.mdx","content/docs/guides/runtime-services/queue-service.mdx","content/docs/guides/runtime-services/settings-service.mdx","content/docs/guides/runtime-services/sharing-service.mdx","content/docs/guides/runtime-services/storage-service.mdx","content/docs/guides/runtime-services/versioning.mdx","content/docs/guides/security.mdx","content/docs/guides/seed-data.mdx","content/docs/guides/single-project-mode.mdx","content/docs/guides/skills.mdx","content/docs/guides/standards.mdx","content/docs/guides/troubleshooting.mdx","content/docs/index.mdx","content/docs/protocol/index.mdx","content/docs/protocol/knowledge.mdx","content/docs/protocol/objectos/config-resolution.mdx","content/docs/protocol/objectos/error-handling.mdx","content/docs/protocol/objectos/http-protocol.mdx","content/docs/protocol/objectos/i18n-standard.mdx","content/docs/protocol/objectos/index.mdx","content/docs/protocol/objectos/lifecycle.mdx","content/docs/protocol/objectos/metadata-service.mdx","content/docs/protocol/objectos/plugin-spec.mdx","content/docs/protocol/objectos/realtime-protocol.mdx","content/docs/protocol/objectos/runtime-capabilities.mdx","content/docs/protocol/objectql/index.mdx","content/docs/protocol/objectql/query-syntax.mdx","content/docs/protocol/objectql/schema.mdx","content/docs/protocol/objectql/security.mdx","content/docs/protocol/objectql/state-machine.mdx","content/docs/protocol/objectql/types.mdx","content/docs/protocol/objectui/actions.mdx","content/docs/protocol/objectui/concept.mdx","content/docs/protocol/objectui/index.mdx","content/docs/protocol/objectui/layout-dsl.mdx","content/docs/protocol/objectui/record-alert.mdx","content/docs/protocol/objectui/widget-contract.mdx","content/docs/releases/index.mdx","content/docs/releases/v9.mdx"] + +const DOCS = (args && Array.isArray(args.docs) && args.docs.length) ? args.docs : ALL_HANDWRITTEN + +const PACKAGE_MAP = `ObjectStack is a metadata-driven application framework. Implementation lives in packages/: +- packages/spec — Zod schemas for every metadata type (.zod.ts); source of truth for shapes & enums. Also packages/spec/src/{data,ui,...}. +- packages/core — kernel: plugin system, service registry, lifecycle, events. +- packages/runtime — runtime services (data, email, queue, settings, sharing, storage, audit, versioning). +- packages/services — service contracts/interfaces (IAuthService, IDataEngine, etc.). +- packages/metadata, metadata-core, metadata-fs — metadata loading/registry (meta.getItem(type, name)). +- packages/cli — the \`os\` CLI; commands in packages/cli/src/commands/ (dev.ts, init.ts, serve.ts, meta/, data/, cloud/, etc.). +- packages/client + client-react — client SDK; public surface in packages/client/src/index.ts. +- packages/rest — REST API; data routes mounted under /api/v1/data/{object}. +- packages/objectql — ObjectQL query engine + types. +- packages/formula — CEL formula engine. +- packages/triggers, observability, mcp, connectors, adapters, platform-objects. +- packages/plugins/* — plugin-auth, plugin-security, plugin-sharing, plugin-approvals, plugin-email, plugin-webhooks, plugin-reports, plugin-audit, plugin-dev, plugin-hono-server, driver-*, knowledge-*, embedder-*. +- apps/console — the admin console app; apps/docs — this docs site.` + +const HOUSE_FACTS = `ESTABLISHED CORRECTIONS from prior doc-accuracy audits (PR #1866 + #1904) — already applied across the docs. Treat as strong priors and apply CONSISTENTLY; re-verify against current code only if a doc's usage looks context-specific: +- CLI binary is \`os\`. There is NO \`os studio\` command — the UI dev command is \`os dev --ui\` (verify in packages/cli/src/commands/dev.ts). +- Metadata access by type+name: \`meta.getItem('object', name)\` — NOT \`client.meta.getObject()\`. +- \`client.ai.chat()\` was REMOVED — do not reference it. +- Approvals are request-id based (ADR-0019): \`client.approvals.*\` — NOT \`client.workflow.approve/reject\`. +- There are NO \`defineProfile\` / \`defineAction\` / \`defineHook\` / \`definePlugin\` helper functions. +- Security model is the real PermissionSet schema (objects/fields, allowCreate/allowRead/allowEdit/allowDelete/..., isProfile) — NOT a Salesforce-style Profile type with objectPermissions/fieldPermissions. FLS non-editable write REJECTS with PermissionDeniedError (403), it is not silently stripped. OWDModel values are public_read / public_read_write. +- Formulas/conditions use CEL — NOT Salesforce UPPERCASE functions. +- Console/portal path is /_console — NOT /_studio. +- REST data path is /api/v1/data/{object} — NOT /api/v1/{object}. REST routes carry the /v1 prefix. +- Env vars are OS_AUTH_SECRET (not AUTH_SECRET) and OS_PORT (not PORT); nested keys use single underscore unless the schema says otherwise. Mock-server toggle is VITE_USE_MOCK_SERVER (no VITE_RUNTIME_MODE or ?mode= switch). +- The repo is github.com/objectstack-ai/framework (NOT objectstack-ai/spec). Fix broken cross-repo links/paths accordingly. +- Package names: @objectstack/ (e.g. @objectstack/service-cache, NOT @objectstack/services/service-cache). Some types only export via subpaths (e.g. @objectstack/spec/ui). +- Auto-generated reference docs live in content/docs/references/ and are OUT OF SCOPE — never edit them.` + +const RULES = `HARD RULES: +1. Edit the doc FILE IN PLACE with Edit/Write. The edits to disk are the real deliverable; your structured output is just a log of what you changed. +2. PRESERVE frontmatter (the --- title/description block) EXACTLY. Do NOT move, rename, or change the file's path or slug. +3. Keep MDX/JSX valid: , , , code fences, import lines must stay well-formed. +4. EVERY factual fix must be backed by evidence you actually read — cite file:line. If you cannot find code confirming a claim is wrong, DO NOT change it. Record it under 'unresolved' instead. +5. Do not fabricate APIs, flags, paths, or features. If the doc describes a feature that does NOT exist in code (removed/aspirational), remove it or qualify it as not-yet-implemented — backed by grep-empty evidence. +6. Make minimal, precise edits — fix what is wrong, leave correct prose alone. +7. Verify code samples, CLI commands, API method names, config keys, env vars, file paths, enum values, and links against the actual implementation.` + +const FIX_LOG_SCHEMA = { + type: 'object', additionalProperties: false, + required: ['doc', 'implementationFound', 'fixesApplied', 'fixCount', 'unresolved', 'notes'], + properties: { + doc: { type: 'string' }, + implementationFound: { type: 'boolean' }, + fixesApplied: { type: 'array', items: { type: 'object', additionalProperties: false, + required: ['category', 'summary', 'before', 'after', 'evidence'], + properties: { + category: { type: 'string', enum: ['broken-example', 'inaccurate-api', 'outdated-path', 'outdated-env', 'security-model', 'fabricated-feature', 'broken-link', 'naming-drift', 'enum-drift', 'other'] }, + summary: { type: 'string' }, before: { type: 'string' }, after: { type: 'string' }, evidence: { type: 'string' }, + } } }, + fixCount: { type: 'number' }, + unresolved: { type: 'array', items: { type: 'string' } }, + notes: { type: 'string' }, + }, +} + +const VERDICT_SCHEMA = { + type: 'object', additionalProperties: false, + required: ['doc', 'fixesReviewed', 'confirmed', 'correctionsMade', 'regressionsFound', 'buildSafe', 'residualInaccuracies'], + properties: { + doc: { type: 'string' }, + fixesReviewed: { type: 'number' }, + confirmed: { type: 'number' }, + correctionsMade: { type: 'array', items: { type: 'object', additionalProperties: false, required: ['what', 'why', 'evidence'], properties: { what: { type: 'string' }, why: { type: 'string' }, evidence: { type: 'string' } } } }, + regressionsFound: { type: 'array', items: { type: 'string' } }, + buildSafe: { type: 'boolean' }, + residualInaccuracies: { type: 'array', items: { type: 'string' } }, + }, +} + +function auditPrompt(doc) { + return `You are auditing a single hand-written ObjectStack documentation file for IMPLEMENTATION ACCURACY and fixing it in place. + +TARGET DOC: ${doc} + +${PACKAGE_MAP} + +${HOUSE_FACTS} + +${RULES} + +PROCEDURE: +1. Read the entire doc (${doc}). +2. For each technical claim — code sample, CLI command, client/server API call, method/type name, config key, enum value, env var, file path, route, link — LOCATE the backing implementation under packages/ (Grep/Glob/Read; ripgrep via Bash is fine) and confirm whether the doc matches reality. +3. Apply evidence-backed fixes directly with Edit. Preserve frontmatter and MDX validity. +4. If a section documents a non-existent/removed/aspirational feature, remove it or qualify it as not-yet-implemented (with grep-empty evidence). +5. Return a structured log of every fix with file:line evidence, plus anything suspected-but-unconfirmed under 'unresolved'. + +A doc with no real inaccuracies should return fixCount 0 — do not invent changes. The edits you write to disk ARE the deliverable.` +} + +function verifyPrompt(doc, fixLog) { + return `You are the ADVERSARIAL VERIFIER for an implementation-accuracy fix just applied to an ObjectStack doc. Assume the previous agent may have over-corrected or introduced errors. + +TARGET DOC (already edited): ${doc} + +FIX LOG from the audit agent (JSON): +${JSON.stringify(fixLog).slice(0, 6000)} + +${PACKAGE_MAP} + +${HOUSE_FACTS} + +${RULES} + +PROCEDURE: +1. Read the current (edited) doc. +2. For EACH applied fix, independently verify the "after" value against the cited evidence AND the live code (re-grep/re-read — do NOT trust the evidence string blindly). +3. Hunt for REGRESSIONS: broken MDX/JSX, altered frontmatter, changed slug, previously-correct content replaced with something wrong, or a NEW inaccuracy. +4. If you find a wrong fix or regression, REPAIR it in place with Edit (same hard rules). Record each repair under correctionsMade with evidence. +5. Confirm frontmatter intact and the file is MDX/build-safe. +6. List remaining suspected inaccuracies under residualInaccuracies (report only; do not fix speculative items). + +Return the verdict.` +} + +phase('Audit & Fix') +log(`Auditing ${DOCS.length} hand-written doc(s) (pipelined: audit -> adversarial verify per doc)`) + +const results = await pipeline( + DOCS, + (doc) => agent(auditPrompt(doc), { label: `audit:${doc.replace('content/docs/', '')}`, phase: 'Audit & Fix', schema: FIX_LOG_SCHEMA }), + (fixLog, doc) => { + if (!fixLog) return null + return agent(verifyPrompt(doc, fixLog), { label: `verify:${doc.replace('content/docs/', '')}`, phase: 'Adversarial Verify', schema: VERDICT_SCHEMA }) + .then((v) => ({ doc, fixLog, verdict: v })) + } +) + +const clean = results.filter(Boolean) +const totalFixes = clean.reduce((n, r) => n + (r.fixLog?.fixCount || 0), 0) +const totalRepairs = clean.reduce((n, r) => n + (r.verdict?.correctionsMade?.length || 0), 0) +const totalResidual = clean.reduce((n, r) => n + (r.verdict?.residualInaccuracies?.length || 0), 0) + +return { + docsProcessed: clean.length, + docsDropped: DOCS.length - clean.length, + docsWithChanges: clean.filter((r) => (r.fixLog?.fixCount || 0) > 0 || (r.verdict?.correctionsMade?.length || 0) > 0).length, + totalFixesApplied: totalFixes, + totalVerifierRepairs: totalRepairs, + totalResidualForFollowup: totalResidual, + docsMissingVerifier: clean.filter((r) => !r.verdict).map((r) => r.doc), + perDoc: clean.map((r) => ({ + doc: r.doc, + fixes: r.fixLog?.fixCount || 0, + implFound: r.fixLog?.implementationFound, + confirmed: r.verdict?.confirmed, + repairs: r.verdict?.correctionsMade?.length || 0, + regressions: r.verdict?.regressionsFound || [], + buildSafe: r.verdict?.buildSafe, + residual: r.verdict?.residualInaccuracies || [], + })), +} diff --git a/.github/workflows/docs-drift-check.yml b/.github/workflows/docs-drift-check.yml new file mode 100644 index 0000000000..f68dcfb354 --- /dev/null +++ b/.github/workflows/docs-drift-check.yml @@ -0,0 +1,80 @@ +name: Docs Drift Check + +# When a PR changes packages/** code, flag the hand-written docs that reference the +# affected packages so they can be re-verified for implementation accuracy before the +# drift lands on main. Advisory only — posts a PR comment, never fails the build. +# The actual LLM audit is run on-demand / on a schedule via the `docs-accuracy-audit` +# workflow, scoped to exactly the docs this check lists. + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'packages/**' + +permissions: + contents: read + pull-requests: write + +jobs: + docs-drift: + name: Flag docs affected by code changes + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Fetch base branch + run: git fetch --no-tags origin "${{ github.base_ref }}" + + - name: Compute affected docs + id: affected + run: | + node scripts/docs-audit/affected-docs.mjs --json "origin/${{ github.base_ref }}" > affected.json + cat affected.json + + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const data = JSON.parse(fs.readFileSync('affected.json', 'utf8')); + const baseRef = context.payload.pull_request.base.ref; + const docs = data.docs || []; + const pkgs = (data.changedPackages || []).map(p => p.name || p.dir); + const marker = ''; + let body; + if (docs.length === 0) { + body = `${marker}\n### 📓 Docs Drift Check\nNo hand-written docs reference the ${pkgs.length} changed package(s). ✅`; + } else { + const detail = (data.detail || []).reduce((m, d) => (m[d.doc] = d.via, m), {}); + const list = docs.map(d => `- \`${d}\`${detail[d] ? ` _(via ${detail[d].join(', ')})_` : ''}`).join('\n'); + body = [ + marker, + '### 📓 Docs Drift Check', + `This PR changes **${pkgs.length}** package(s): ${pkgs.map(p => `\`${p}\``).join(', ')}.`, + '', + `**${docs.length}** hand-written doc(s) reference the affected code and may need an implementation-accuracy re-verification:`, + '', + list, + '', + '> Advisory only. To re-verify, run the `docs-accuracy-audit` workflow scoped to these files:', + '> `node scripts/docs-audit/affected-docs.mjs origin/' + baseRef + '` → pass the list as `args.docs`.', + ].join('\n'); + } + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); + } diff --git a/scripts/docs-audit/README.md b/scripts/docs-audit/README.md new file mode 100644 index 0000000000..85b3f055ae --- /dev/null +++ b/scripts/docs-audit/README.md @@ -0,0 +1,71 @@ +# Docs accuracy verification + +Keeps the **hand-written** docs (`content/docs/**` minus `content/docs/references/**`) +in sync with the actual implementation in `packages/**` as the platform evolves. +Generated references (`content/docs/references/`) are produced from `packages/spec` +and are out of scope here — regenerate those separately. + +The system has four parts, layered cheapest-and-earliest first: + +## 1. `affected-docs.mjs` — change → docs mapping (the linchpin) + +Maps a set of `packages/**` changes to the hand-written docs that reference the +affected packages, so an audit can be scoped to what actually changed. + +```bash +# docs affected by changes on this branch vs origin/main +node scripts/docs-audit/affected-docs.mjs origin/main + +# JSON (with changed packages + per-doc "why") +node scripts/docs-audit/affected-docs.mjs --json origin/main + +# every hand-written doc (full audit scope) +node scripts/docs-audit/affected-docs.mjs --all +``` + +Heuristic: a doc is *affected* by a changed package `P` if it mentions `P`'s npm +name (`@objectstack/`) or repo path (`packages/`). Over-inclusion is preferred +over misses; the periodic **full** audit (part 4) is the backstop for docs that +describe a package without naming it. + +## 2. CI gate — `.github/workflows/docs-drift-check.yml` + +On any PR that touches `packages/**`, runs `affected-docs.mjs` against the base branch +and posts/updates a single advisory PR comment listing the docs that reference the +changed code. **Never fails the build** — it only flags drift at the source, before it +lands on `main`. Reviewers (or an on-demand audit run) decide whether to re-verify. + +## 3. `docs-accuracy-audit` workflow — the LLM audit + +A Claude Code multi-agent workflow (`.claude/workflows/docs-accuracy-audit.js`). For each +doc: an agent reads it, locates the real implementation, and applies evidence-backed +fixes in place; a second **adversarial verifier** re-checks every fix against the code and +repairs over-corrections. Scope it with `args.docs`; omit for a full audit. + +```js +// scoped to the docs a code change touched: +Workflow({ name: 'docs-accuracy-audit', args: { docs: [/* output of affected-docs.mjs */] } }) +// full audit of all hand-written docs: +Workflow({ name: 'docs-accuracy-audit' }) +``` + +It edits files in place (frontmatter preserved, no moves) and returns a per-doc log of +fixes, verifier repairs, and residual items that couldn't be confirmed against code. +Always follow a run with the docs build gate: + +```bash +pnpm --filter @objectstack/docs build # must compile all pages clean +``` + +## 4. Scheduled routine — periodic backstop + +A cron routine (created via the `schedule` skill) runs on a cadence (default monthly / +per-release) to catch drift the CI gate missed. It computes the change-scoped doc list +since the last audit, runs the `docs-accuracy-audit` workflow on it, runs the build, and +opens a PR when there are fixes. See the routine prompt for the exact steps. + +--- + +**Cost note:** a full audit of all 128 hand-written docs is ~2.8M output tokens / ~160 +agents. Always prefer the change-scoped list (`affected-docs.mjs`) over `--all` except for +the periodic full backstop. diff --git a/scripts/docs-audit/affected-docs.mjs b/scripts/docs-audit/affected-docs.mjs new file mode 100644 index 0000000000..e39c1cbbb5 --- /dev/null +++ b/scripts/docs-audit/affected-docs.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +// Map a set of `packages/**` code changes to the hand-written docs that reference +// the affected packages, so a doc-accuracy audit can be scoped to what actually +// changed instead of re-auditing all 128 hand-written docs every time. +// +// Usage: +// node scripts/docs-audit/affected-docs.mjs [sinceRef] # docs affected by changes since (default origin/main) +// node scripts/docs-audit/affected-docs.mjs --all # every hand-written doc (full audit) +// node scripts/docs-audit/affected-docs.mjs --json [...] # emit JSON {docs, changedPackages, ...} instead of a path list +// +// Scope: hand-written docs only = content/docs/**/*.mdx MINUS content/docs/references/** +// (references are generated from packages/spec and handled by a separate regenerate pass). +// +// Heuristic: a doc is "affected" by a changed package P if the doc text mentions P's +// npm name (`@objectstack/`) or its repo path (`packages/`). Over-inclusion is +// intentionally preferred over misses; the periodic FULL audit is the backstop for +// docs that describe a package without naming it. + +import { execSync } from 'node:child_process'; +import { readFileSync, readdirSync, existsSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; + +const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim(); +const args = process.argv.slice(2); +const asJson = args.includes('--json'); +const all = args.includes('--all'); +const sinceRef = args.find((a) => !a.startsWith('--')) || 'origin/main'; + +function sh(cmd) { + return execSync(cmd, { cwd: repoRoot, stdio: ['ignore', 'pipe', 'ignore'] }).toString(); +} + +// --- 1. enumerate hand-written docs ---------------------------------------- +function walk(dir, out = []) { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) walk(p, out); + else if (e.isFile() && p.endsWith('.mdx')) out.push(p); + } + return out; +} +const docsRoot = join(repoRoot, 'content/docs'); +const refsRoot = join(repoRoot, 'content/docs/references'); +const handwritten = walk(docsRoot) + .filter((p) => !p.startsWith(refsRoot)) + .map((p) => relative(repoRoot, p)) + .sort(); + +if (all) { + emit(handwritten, [], 'all hand-written docs'); + process.exit(0); +} + +// --- 2. changed package roots since ----------------------------- +let changedFiles = []; +try { + // three-dot: changes on HEAD since the merge-base with sinceRef + changedFiles = sh(`git diff --name-only ${sinceRef}...HEAD -- packages/`).split('\n').filter(Boolean); +} catch { + // fall back to two-dot (e.g. detached/ranges that lack a merge-base) + changedFiles = sh(`git diff --name-only ${sinceRef} -- packages/`).split('\n').filter(Boolean); +} + +// collect package roots: packages/ and packages/plugins/ +const pkgRoots = new Set(); +for (const f of changedFiles) { + let m = f.match(/^(packages\/plugins\/[^/]+)\//) || f.match(/^(packages\/[^/]+)\//); + if (m) pkgRoots.add(m[1]); +} + +// resolve each root to its npm name + keep the path token +const changedPackages = []; // {dir, name} +for (const dir of pkgRoots) { + let name = null; + const pj = join(repoRoot, dir, 'package.json'); + if (existsSync(pj)) { + try { name = JSON.parse(readFileSync(pj, 'utf8')).name || null; } catch { /* ignore */ } + } + changedPackages.push({ dir, name }); +} + +// --- 3. match docs that mention an affected package ------------------------ +const affected = []; +for (const doc of handwritten) { + const text = readFileSync(join(repoRoot, doc), 'utf8'); + const hits = []; + for (const { dir, name } of changedPackages) { + if (name && text.includes(name)) hits.push(name); + else if (text.includes(dir)) hits.push(dir); + } + if (hits.length) affected.push({ doc, via: [...new Set(hits)] }); +} + +emit(affected.map((a) => a.doc), changedPackages, `${affected.length} docs affected by ${changedPackages.length} changed package(s) since ${sinceRef}`, affected); + +function emit(docList, changedPackages, summary, detail) { + if (asJson) { + process.stdout.write(JSON.stringify({ summary, sinceRef: all ? null : sinceRef, changedPackages, docs: docList, detail: detail || null }, null, 2) + '\n'); + } else { + process.stderr.write(`# ${summary}\n`); + process.stdout.write(docList.join('\n') + (docList.length ? '\n' : '')); + } +}