feat: CLI telemetry for all commands with centralized lifecycle

CLAUDE.md

@@ -48,6 +48,39 @@ pnpm typecheck # Type check
 2. Register in `src/bin.ts` and update `src/utils/help-json.ts` command registry
 3. Include JSON mode tests in spec file
 
+## Telemetry Wiring for New Commands
+
+All commands automatically emit a `command` telemetry event with name, duration, and success/failure. The centralized lifecycle in `bin.ts` (`runCli()`) handles this — no manual wrapping required.
+
+**Subcommands via `registerSubcommand()`** — auto-tracked. Just write the handler:
+
+```typescript
+.command('user', 'Manage users', (yargs) => {
+ registerSubcommand(yargs, 'reset-password', '...', (y) => y,
+ async (argv) => { await runResetPassword(argv); },
+ );
+})
+```
+
+**Top-level `.command()` with inline handler** — also auto-tracked:
+
+```typescript
+.command(
+ 'migrate',
+ 'Migrate from another provider',
+ (yargs) => yargs.options({...}),
+ async (argv) => {
+ await runMigrate(argv);
+ },
+)
+```
+
+**Exiting with errors:** Use `exitWithError()` or `exitWithCode()` from handlers — they throw `CliExit` which the lifecycle catches, classifies, and records.
+
+**Skip list**: Commands in `SKIP_TELEMETRY_COMMANDS` (`command-telemetry.ts`) are excluded from command-level telemetry because they have their own session-based telemetry. Currently: `install`, `dashboard`, `root` (the default `$0` handler).
+
+**Aliases**: if you register a command with multiple names (e.g., `['organization', 'org']`), add the alias to `src/lib/command-aliases.ts` so metrics don't fragment.
+
 ## Do / Don't
 
 **Do:**

README.md

@@ -736,17 +736,18 @@ OAuth credentials are stored in the system keychain (with `~/.workos/credentials
 
 ## Telemetry
 
-The installer collects anonymous usage telemetry to help improve the product:
+The CLI collects anonymous usage telemetry to help improve the product:
 
-- Session outcome (success/error/cancelled)
-- Framework detected
-- Duration and step timing
-- Token usage (for capacity planning)
+- **Command events** -- command name, duration, success/failure, termination reason, and which flags were used (for telemetry-enabled commands; `install` and `dashboard` use session events instead)
+- **Session events** -- framework detected, step timing, token usage (installer only)
+- **Crash events** -- sanitized error type and stack trace (no secrets, truncated to 4KB)
 
-No code, credentials, or personal data is collected. Disable with:
+Environment fingerprint (OS, Node version, shell, CI detection) is included on all events. No code, credentials, or personal data is collected.
+
+Disable with:
 
 ```bash
-WORKOS_TELEMETRY=false npx workos@latest install
+WORKOS_TELEMETRY=false workos <command>
 ```
 
 ## Logs

src/bin-command-telemetry.integration.spec.ts

@@ -0,0 +1,122 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/**
+ * Integration test for the command-telemetry lifecycle in bin.ts.
+ *
+ * bin.ts runs runCli() at import and exposes no seams, so the only honest way
+ * to test the wiring is to drive the real CLI as a subprocess. We run a KNOWN
+ * command with a missing required argument and assert that the resulting
+ * `command` telemetry event is recorded (attributed to the real command, with
+ * a validation_error reason) rather than being silently dropped.
+ *
+ * Regression guard for the gap where yargs runs its demand/strict validation
+ * before dispatching middleware: a validation failure short-circuited before
+ * the command-name middleware ran, leaving the name as 'root' (which is in
+ * SKIP_TELEMETRY_COMMANDS), so we lost telemetry for every misused command.
+ *
+ * We observe the emitted event via the store-forward pending file rather than
+ * stdout: pointing the CLI at an unroutable telemetry URL makes the flush fail,
+ * so the queued events are persisted to <TMPDIR>/workos-cli-telemetry/ on exit.
+ * That captures the real event payload, independent of debug-log formatting.
+ */
+const binPath = fileURLToPath(new URL('./bin.ts', import.meta.url));
+const forceInsecureStorageImport = new URL('./test/force-insecure-storage.ts', import.meta.url).href;
+const repoRoot = fileURLToPath(new URL('..', import.meta.url));
+
+let sandboxTmp: string;
+
+beforeEach(() => {
+ sandboxTmp = mkdtempSync(join(tmpdir(), 'wos-cli-telem-it-'));
+});
+
+afterEach(() => {
+ rmSync(sandboxTmp, { recursive: true, force: true });
+});
+
+function runCli(args: string[]) {
+ const env: NodeJS.ProcessEnv = {
+ PATH: process.env.PATH,
+ HOME: sandboxTmp,
+ USERPROFILE: sandboxTmp,
+ TMPDIR: sandboxTmp,
+ TMP: sandboxTmp,
+ TEMP: sandboxTmp,
+ // Keep prompts/update checks disabled without inheriting host agent/CI env.
+ WORKOS_MODE: 'agent',
+ // Force telemetry on so a host WORKOS_TELEMETRY=false can't make the test
+ // silently produce no event and fail.
+ WORKOS_TELEMETRY: 'true',
+ // Unroutable URL: the flush fails, so the queued events are persisted to
+ // the pending file on exit where we can inspect the real payload.
+ WORKOS_TELEMETRY_URL: 'http://127.0.0.1:59999/cli',
+ WORKOS_API_KEY: 'sk_dummy_for_test',
+ };
+
+ const result = spawnSync(
+ process.execPath,
+ ['--import', 'tsx', '--import', forceInsecureStorageImport, binPath, ...args],
+ {
+ cwd: repoRoot,
+ encoding: 'utf-8',
+ env,
+ },
+ );
+
+ const events: Array<{ type: string; attributes?: Record<string, unknown> }> = [];
+ const pendingDir = join(sandboxTmp, 'workos-cli-telemetry');
+ for (const file of readdirSync(pendingDir, { withFileTypes: true })) {
+ if (file.isFile() && file.name.startsWith('pending-') && file.name.endsWith('.json')) {
+ events.push(...JSON.parse(readFileSync(join(pendingDir, file.name), 'utf-8')));
+ }
+ }
+ return { result, events };
+}
+
+describe('command telemetry lifecycle', () => {
+ it('emits a command event for a known command that fails validation', () => {
+ // `organization create` requires a positional `name`; omitting it is a
+ // validation error on a real, known command.
+ const { result, events } = runCli(['organization', 'create']);
+
+ expect(`${result.stdout}\n${result.stderr}`).toContain('Not enough non-option arguments');
+
+ const command = events.find((e) => e.type === 'command');
+ expect(command).toBeDefined();
+ // Validation fails before middleware runs, so only the top-level command is
+ // recovered (subcommand precision is intentionally dropped to avoid leaking
+ // positional values). The key regression: it's attributed, not skipped as 'root'.
+ expect(command?.attributes?.['command.name']).toBe('organization');
+ expect(command?.attributes?.['termination.reason']).toBe('validation_error');
+ expect(command?.attributes?.['command.success']).toBe(false);
+ }, 20_000);
+
+ it('records a crash event with a redacted stack when a command crashes unexpectedly', () => {
+ // `debug simulate --crash` throws a plain Error (not CliExit), which the
+ // lifecycle must classify as a crash (not validation_error).
+ const { result, events } = runCli(['debug', 'simulate', '--crash']);
+ expect(result.status).not.toBe(0);
+
+ // A crash must not be a silent exit-1: the error surfaces on stderr.
+ expect(`${result.stdout}\n${result.stderr}`).toMatch(/Simulated crash/);
+
+ const crash = events.find((e) => e.type === 'crash');
+ expect(crash).toBeDefined();
+ expect(crash?.attributes?.['crash.error_type']).toBe('Error');
+ expect(crash?.attributes?.['crash.command']).toBe('debug.simulate');
+
+ // The accompanying command event is classified as a crash, not a validation error.
+ const command = events.find((e) => e.type === 'command');
+ expect(command?.attributes?.['termination.reason']).toBe('crash');
+
+ // Stack must be redacted: no absolute home path, no full repo path.
+ const stack = String(crash?.attributes?.['crash.stack'] ?? '');
+ expect(stack).toContain('Simulated crash');
+ expect(stack).not.toMatch(/\/Users\/[^/]+\//); // POSIX home dir collapsed to ~
+ expect(stack).not.toContain(repoRoot);
+ }, 20_000);
+});