diff --git a/docs/qa/platform-checklist/areas/cli.json b/docs/qa/platform-checklist/areas/cli.json index aa608cf2bb..1cdcdd5f95 100644 --- a/docs/qa/platform-checklist/areas/cli.json +++ b/docs/qa/platform-checklist/areas/cli.json @@ -1,7 +1,7 @@ { "$comment": "Standing platform test checklist — area ledger. Hand-edited, append-only; validated by scripts/check-platform-checklist.mjs (pnpm check:platform-checklist). Authoring rules: docs/qa/platform-checklist/README.md · execution protocol: RUNNER.md.", "area": "cli", - "title": "CLI — os dev/build/migrate/verify boot & exit-code contracts, scaffold first-run, flag/command error UX", + "title": "CLI — os dev/build/migrate/verify/test boot, load, execution & exit-code contracts, scaffold first-run, flag/command error UX", "items": [ { "id": "cli.dev-boot-contract", @@ -427,6 +427,129 @@ { "revision": 1, "date": "2026-08-07", "change": "new item: the published first-run experience mirrored step-for-step from scaffold-e2e.yml, template matrix as variants, RC-window protocol refusal recorded as gate-working instead of failure", "ref": "claude/platform-test-checklist-ocwugl" } ] }, + { + "id": "cli.qa-suite-execution", + "title": "os test: a Quality Protocol suite is validated at LOAD, executed against a booted app, and its verdict is the exit code — capture/interpolation thread state, an unevaluable assertion FAILS", + "since": "v17", + "status": "active", + "revision": 1, + "priority": "P1", + "surface": "cli", + "personas": ["operator (local shell)", "suite author (writes qa/*.test.json)"], + "fixtures": { + "app": "showcase", + "requires": [ + "a free non-default port and an ephemeral boot: `pnpm dev -- --fresh -p ` from the repo root (showcase + --seed-admin; dogfood skill §0)", + "a session token for the data steps — POST /api/v1/auth/sign-in/email with the seeded admin (admin@objectos.ai / admin123) and read `.token`; pass it as `os test --token ` (the CLI sends it as `Authorization: Bearer`)", + "the shipped fixture suite examples/app-showcase/qa/platform-smoke.test.json — the repo's only Quality Protocol suite, authored so `os test`'s DEFAULT glob (`qa/*.test.json`) finds it from that directory with no argument", + "a scratch directory with its own qa/ for the negative and variant probes — deliberately broken suites are STAGED per run, never committed, so the repo's own suite stays green" + ], + "knownGaps": [ + "five of the eight action types cannot reach a stock server at all (see `negative`): `HttpTestAdapter` builds `${baseUrl}/api/data/:object` while the server serves `{basePath}/data/:object` with basePath `/api/v1`. The variant sweep therefore records five refusals, and that is the honest verdict — not a fixture gap to work around by rewriting the probes as `api_call`", + "no scenario SELECTION exists: `os test` has exactly two flags (--url, --token), `scenario.tags` filters nothing and `scenario.requires` is never checked (packages/spec/liveness/qa.json rows `tags`/`requires`), so the whole glob always runs and a suite cannot declare a precondition it will be skipped for" + ] + }, + "steps": [ + "boot the app: from the repo root `pnpm dev -- --fresh -p `; poll `curl -s -m3 -o /dev/null -w '%{http_code}' http://localhost:/api/v1/health` until 200", + "mint a token: `curl -s -X POST http://localhost:/api/v1/auth/sign-in/email -H 'Content-Type: application/json' -H 'Origin: http://localhost:' -d '{\"email\":\"admin@objectos.ai\",\"password\":\"admin123\"}'` and read `.token`", + "the green path on the DEFAULT glob: `cd examples/app-showcase && os test --url http://localhost: --token ; echo $?` — capture the whole transcript including the `Found N test suites.` line and the summary", + "prove the teardown ran: list `GET /api/v1/data/showcase_category` before and after and confirm the row the run created is gone (the run leaves the fixture exactly as it found it)", + "prove the run repeats: execute the same command a second time and confirm it is green again with no leftover row — a suite that only passes once is a suite whose teardown is decorative", + "stage the negative probes in a scratch dir's qa/: (a) a suite with `steps` misspelled as `stpes`, (b) a suite asserting `data.status equals degraded` against /api/v1/health, (c) a suite asserting `contains` over `data.nope`, a path the health response does not carry. Run `os test 'qa/*.test.json' --url http://localhost:; echo $?` and capture all three refusals in one transcript", + "prove the malformed suite never ran: the server log carries no request attributable to it, and the transcript shows the two OTHER staged suites still executing after the refusal", + "teardown-after-failure: stage a scenario that creates a record, captures its id, then fails a deliberate assertion, with a teardown that deletes `{{id}}`; run it and confirm via the API that the record is gone despite the red verdict", + "variant sweep: stage one scenario per TestActionTypeSchema member (8) and run them in one suite; record each member's own verdict and the exact error text of every refusal — no member inferred from a sibling", + "zero-match glob: `os test 'qa/nothing-matches-*.test.json' --url http://localhost:; echo $?` and record BOTH the message and the exit code", + "glob discipline (#7363): `os test 'qa/**/*.test.json'` from the app dir — it must complete in seconds and find the same suite, never the multi-minute walk that preceded the lazy resolver", + "persona check: re-run the green suite with NO --token and record which scenario survives (health) and which fails (the data roundtrip, HTTP 401)" + ], + "acceptance": [ + { + "clause": "the green path closes: every scenario reports ✅ with its scenarioId and a duration, the summary reads `SUCCESS: All N scenarios passed.`, and the process exits 0", + "oracle": "log", + "verify": "the transcript shows one ✅ line per scenario and `echo $?` prints 0 — AND the evidence quotes the `Found N test suites.` line with N > 0, because a run that discovered nothing also exits 0 (see `negative`)", + "evidence": "the full os test transcript + exit code" + }, + { + "clause": "state threads through the run: a step's `capture` writes into the scenario context and `{{var}}` interpolation reads it back into a LATER step's action, so the read-back step addresses the record the create step actually made", + "oracle": "api", + "verify": "the roundtrip scenario passes AND the id it captured is confirmed independently — list/GET the object over the API and match the record the run created; a green scenario whose URL still contains the literal `{{categoryId}}` would be interpolation dead and the step failing for the wrong reason", + "evidence": "the scenario transcript + the independent API read of the same id" + }, + { + "clause": "a false assertion FAILS the scenario and the run: the scenario is ❌, the failing step is named, the message names the field, the expected and the actual value, and the process exits 1", + "oracle": "log", + "verify": "the staged `equals degraded` probe prints `Assertion failed: data.status expected degraded, got ok` under `Step Failed: `; `echo $?` prints 1", + "evidence": "the probe transcript + exit code" + }, + { + "clause": "#7256 — an assertion the engine CANNOT evaluate fails, it does not pass: `contains` against a path the response does not carry names the runtime type it found and says which of the fixture or the assertion is the suspect", + "oracle": "log", + "verify": "the `contains` over `data.nope` probe fails with `cannot be evaluated by 'contains' … got undefined` plus the `is_null` hint — a ✅ here is the exact silent-pass regression the clause exists for", + "evidence": "the probe transcript" + }, + { + "clause": "#6247 — load-site enforcement holds: a document TestSuiteSchema rejects is refused BEFORE execution, naming the file, every offending path, the expected shape and the doc reference; it counts as one failed suite; the rest of the glob still runs; the run exits 1", + "oracle": "log", + "verify": "the malformed probe prints ` is not a valid Quality Protocol suite (TestSuiteSchema)` with `✗ scenarios.0.steps: …`, the two sibling suites still execute after it, and the summary counts it as failed — never `SUCCESS` and never a TypeError attributed to the runner", + "evidence": "the three-suite transcript + exit code" + }, + { + "clause": "teardown runs even after a failed step, so a red run does not leave the fixture dirty — and a passing teardown never rescues an already-failed scenario", + "oracle": "api", + "verify": "after the staged fail-then-teardown scenario the record created in step 1 is absent from the object listing, while the scenario's verdict stays ❌", + "evidence": "the scenario verdict + the before/after listings" + }, + { + "clause": "every one of the 8 TestActionTypeSchema members carries its OWN recorded verdict — executed and asserted, or refused with a named error — and none is inferred from a sibling", + "oracle": "log", + "verify": "the sweep transcript shows all 8 scenario ids with their individual outcomes; refusals are quoted verbatim (the adapter's `Unsupported action type in HttpAdapter: ` and any HTTP error) rather than summarised as 'the record actions don't work'", + "evidence": "the 8-scenario sweep transcript" + }, + { + "clause": "#7363 glob discipline: the default `qa/*.test.json` resolves relative to the invocation directory, a `**` pattern completes in seconds, and matches run in sorted order", + "oracle": "log", + "verify": "the bare `os test` from examples/app-showcase finds the shipped suite with no argument; the quoted `qa/**/*.test.json` run finds the same suite and returns in seconds — the pre-#7363 resolver died at exit 134 after ~7 minutes on this monorepo", + "evidence": "both transcripts with wall-clock timings" + } + ], + "negative": [ + "a zero-match glob prints `No test files found matching: ` and exits **0** (measured 2026-08-11) — a green exit from a run that loaded no suite. A run record that ticks this item without quoting a `Found N test suites.` line with N > 0 is the false positive this item exists to prevent, and a CI job wired to `os test` with a typo'd path would report success forever", + "a malformed suite that REACHES the runner — reported passed having executed nothing, or dying as a TypeError inside runSuite with no file named — is the #6247 regression", + "a `contains` against a path the result does not carry reporting ✅ is the #7256 regression", + "the five record-shaped action types (`create_record`, `read_record`, `update_record`, `delete_record`, `query_records`) answer **HTTP 404** against a stock server: `HttpTestAdapter` builds `${baseUrl}/api/data/:object` while the server serves `{basePath}/data/:object` with basePath `/api/v1` (and `update_record` issues PUT where the route is PATCH). Measured 2026-08-11 on showcase. A run must record those four-oh-fours as the measured verdict — recording them as 'not applicable' hides the finding, and a future run that finds them PASSING means the adapter was repaired, which is a revision of this item rather than a silent green", + "satisfying this item by executing the unit pins in `source` instead of driving a booted app is not a run: those pins cover the load boundary, the glob resolver and one assertion operator, and none of them proves a suite reaches a real server — which is why this item carries no `automated` entry" + ], + "variants": [ + "api_call — the only action type that reaches the real API surface (target is a path or absolute URL; `payload.method`/`payload.body` shape the request)", + "wait — local, no HTTP; resolves `{ waited: }` from `payload.duration`", + "create_record — refused: POST ${baseUrl}/api/data/:object → 404 (see `negative`)", + "read_record — refused: GET ${baseUrl}/api/data/:object/:id → 404", + "update_record — refused: PUT ${baseUrl}/api/data/:object/:id → 404 (and PUT, where the route is PATCH)", + "delete_record — refused: DELETE ${baseUrl}/api/data/:object/:id → 404", + "query_records — refused: POST ${baseUrl}/api/data/:object/query → 404", + "run_script — declared in the enum with NO adapter branch: `Unsupported action type in HttpAdapter: run_script` (loud, recorded in packages/spec/liveness/qa.json)" + ], + "enumSource": { + "file": "packages/spec/src/qa/testing.zod.ts", + "export": "TestActionTypeSchema", + "expect": 8 + }, + "traps": ["dispatcher-vs-hono-route", "stale-dist", "wrong-persona"], + "source": [ + "packages/cli/src/commands/test.ts (the shipped `os test`: the #7363 lazy segment-directed glob with its prune list, `loadTestSuite`'s #6247 boundary parse, the per-scenario report, and the exit 0/1 summary)", + "packages/core/src/qa/runner.ts (scenario sequencing, `capture` + `{{var}}` interpolation, the assertion operators, setup/teardown semantics, the #7256 unevaluable-`contains` fix)", + "packages/core/src/qa/http-adapter.ts (the action-type switch — its case labels ARE the enum values; the `/api/data/...` record routes this item's variants measure against the server's `/api/v1/data/...`)", + "packages/spec/src/qa/testing.zod.ts (TestSuiteSchema — the shape enforced at load; TestActionTypeSchema pinned above)", + "packages/spec/liveness/qa.json (the ADR-0049 ledger whose existence this item is coverage.json's mapping for — its dead `tags`/`requires` rows are why no scenario selection exists)", + "packages/cli/test/qa-suite-schema-load.test.ts, packages/cli/test/resolve-glob-lazy-walk.test.ts, packages/core/src/qa/runner.test.ts (the three unit pins — cited so a run knows what is already covered, NOT a substitute for driving a booted app)", + "content/docs/deployment/cli.mdx §os test (the documented command contract)", + "examples/app-showcase/qa/platform-smoke.test.json (the fixture suite this item drives)" + ], + "history": [ + { "revision": 1, "date": "2026-08-11", "change": "new item: the `qa` capability's coverage.json mapping, authored rather than waived (#7347 triage ruling). `os test` is a shipped, documented CLI command, so the honest mapping is a surface:cli item that authors a real qa/*.test.json suite and drives it against a booted app — the fixture suite examples/app-showcase/qa/platform-smoke.test.json lands with this item and is the repo's first Quality Protocol suite. Every clause was measured on showcase before it was written: the green path, capture/interpolation, the #6247 load refusal, the #7256 unevaluable-contains failure, teardown-after-failure, the 8-member action-type sweep and the #7363 glob. `since: v17` records the release in which the surface became GOVERNED (liveness ledger seeded + TestSuiteSchema enforced at the load site, #6247 / PR #7255); the command itself predates it. No `automated` entry: the three unit pins cover pieces, none of them proves a suite reaches a real server", "ref": "#7347" } + ] + }, { "id": "cli.flag-command-error-ux", "title": "Wrong flags and unknown commands error with usage and a nonzero exit — never silently ignored, never executed anyway", diff --git a/docs/qa/platform-checklist/coverage.json b/docs/qa/platform-checklist/coverage.json index f039596adb..34070f0a61 100644 --- a/docs/qa/platform-checklist/coverage.json +++ b/docs/qa/platform-checklist/coverage.json @@ -124,6 +124,11 @@ "identity-auth.teams-bu-membership" ] }, + "qa": { + "items": [ + "cli.qa-suite-execution" + ] + }, "query": { "items": [ "api-backend.query-contract-matrix" diff --git a/examples/app-showcase/qa/platform-smoke.test.json b/examples/app-showcase/qa/platform-smoke.test.json new file mode 100644 index 0000000000..956ab7fb87 --- /dev/null +++ b/examples/app-showcase/qa/platform-smoke.test.json @@ -0,0 +1,92 @@ +{ + "name": "Showcase platform smoke (Quality Protocol)", + "scenarios": [ + { + "id": "health-liveness", + "name": "The booted server answers its liveness probe", + "description": "The cheapest real assertion in the suite, and the one that tells a failing run whether the target URL was even right. GET /api/v1/health is the unauthenticated liveness probe (packages/runtime/src/http-dispatcher.ts), so this scenario passes with or without --token — a failure here means the --url is wrong or the server is down, not that the data assertions below are broken.", + "steps": [ + { + "name": "GET /api/v1/health", + "action": { + "type": "api_call", + "target": "/api/v1/health", + "payload": { "method": "GET" } + }, + "assertions": [ + { "field": "success", "operator": "equals", "expectedValue": true }, + { "field": "data.status", "operator": "equals", "expectedValue": "ok" }, + { "field": "data.timestamp", "operator": "not_null", "expectedValue": null } + ] + } + ] + }, + { + "id": "category-create-read-roundtrip", + "name": "A record created through the data API reads back by the id the create returned", + "description": "The load-bearing scenario: it can only pass if capture, {{variable}} interpolation and assertion evaluation all work, because the read step's URL is built from the id the create step captured. showcase_category is the target because it is sharingModel: public_read_write with exactly one required field (examples/app-showcase/src/data/objects/category.object.ts), so the record is legal on stock fixtures. Requires --token (a signed-in session); the data routes enforce auth.", + "tags": ["smoke", "crud"], + "steps": [ + { + "name": "create a category", + "description": "POST /api/v1/data/:object — the wire body IS the record (#3899).", + "action": { + "type": "api_call", + "target": "/api/v1/data/showcase_category", + "payload": { + "method": "POST", + "body": { "name": "QA smoke — Quality Protocol roundtrip" } + } + }, + "capture": { "categoryId": "id" }, + "assertions": [ + { "field": "id", "operator": "not_null", "expectedValue": null }, + { "field": "object", "operator": "equals", "expectedValue": "showcase_category" }, + { "field": "record.name", "operator": "equals", "expectedValue": "QA smoke — Quality Protocol roundtrip" } + ] + }, + { + "name": "read it back by the captured id", + "description": "The target carries {{categoryId}}: if capture or interpolation is dead, this GET asks for the literal path segment and the step fails instead of silently passing.", + "action": { + "type": "api_call", + "target": "/api/v1/data/showcase_category/{{categoryId}}", + "payload": { "method": "GET" } + }, + "assertions": [ + { "field": "record.id", "operator": "not_null", "expectedValue": null }, + { "field": "record.name", "operator": "equals", "expectedValue": "QA smoke — Quality Protocol roundtrip" }, + { "field": "record.parent", "operator": "is_null", "expectedValue": null } + ] + }, + { + "name": "find it in the list response", + "description": "`contains` over an array — the operator whose non-evaluable shapes used to pass silently (#7256). The field path points at a real array so the assertion is evaluated, not skipped.", + "action": { + "type": "api_call", + "target": "/api/v1/data/showcase_category", + "payload": { "method": "GET" } + }, + "assertions": [ + { "field": "records.0.id", "operator": "not_null", "expectedValue": null }, + { "field": "object", "operator": "contains", "expectedValue": "category" } + ] + } + ], + "teardown": [ + { + "name": "delete the category", + "description": "Teardown runs even after a failed step, so a red run does not leave the fixture dirty for the next one.", + "action": { + "type": "api_call", + "target": "/api/v1/data/showcase_category/{{categoryId}}", + "payload": { "method": "DELETE" } + }, + "assertions": [ + { "field": "success", "operator": "equals", "expectedValue": true } + ] + } + ] + } + ] +}