test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
UncertaintyDeterminesYou4ndMe wants to merge 1 commit into from
Closed

test(headless): run ordinary CLI command semantics in process #2479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 61 additions & 22 deletions packages/headless/src/__tests__/cli.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
args: string[],
options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
return new Promise((resolve) => {
const child = spawn(process.execPath, [cliPath, ...args], {
env: { ...process.env, ...options.env },
env: { ...process.env },
});
let stdout = '';
let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
});
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
args: string[],
options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
const mapped = mapLegacyMakaHeadlessArgs(args);
assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
const envOverrides = Object.entries(options.env ?? {});
const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
for (const [key, value] of envOverrides) process.env[key] = value;
const savedExitCode = process.exitCode;
try {
const {
result: code,
stdout,
stderr,
} = await withCapturedProcessIo(async () => {
try {
return await runMakaEvalCli(mapped);
} catch (error) {
// Mirror the bin's fatal handler: report the error and fail with 1.
process.stderr.write(
`${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
);
return 1;
}
});
return { code, stdout, stderr };
} finally {
process.exitCode = savedExitCode;
for (const [key, value] of savedEnv) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
}

describe('maka-headless CLI', () => {
test('task-run readers report an actionable error for a non-Headless root', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
};
const specPath = join(dir, 'spec.json');
await writeFile(specPath, JSON.stringify(spec), 'utf8');
const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
assert.equal(result.code, 1);
assert.match(result.stderr, /protectedPaths/);
} finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
try {
await mkdir(join(dir, 'fixture'), { recursive: true });
const result = await runCli([
const result = await runCliInProcess([
'harbor',
'run',
'--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
try {
await mkdir(join(dir, 'fixture'), { recursive: true });
const missingUrl = await runCli([
const missingUrl = await runCliInProcess([
'harbor',
'run',
'--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
assert.equal(missingUrl.code, 1);
assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

const missingToken = await runCli(
const missingToken = await runCliInProcess(
[
'harbor',
'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
await mkdir(fixture, { recursive: true });
await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

const result = await runCli(
const result = await runCliInProcess(
[
'harbor',
'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
await mkdir(fixture, { recursive: true });
await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

const result = await runCli(
const result = await runCliInProcess(
[
'harbor',
'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
};
const specPath = join(dir, 'spec.json');
await writeFile(specPath, JSON.stringify(spec), 'utf8');
const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
assert.equal(result.code, 0, result.stderr);
const records = await readResults(join(dir, 'out', 'results.jsonl'));
assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
const outDir = join(dir, 'out');
await writeFile(specPath, JSON.stringify(spec), 'utf8');

const run = await runCli([
const run = await runCliInProcess([
'task',
'run',
specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
assert.equal(run.code, 1);
assert.match(run.stdout, /taskRunId: task-run-1/);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
assert.ok(Array.isArray(inspectDocument.attempts));

const humanInspect = await runCli([
const humanInspect = await runCliInProcess([
'task',
'inspect',
'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

const exportDir = join(dir, 'manual-export');
const exported = await runCli([
const exported = await runCliInProcess([
'task',
'export',
'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
}

const aheExportDir = join(dir, 'ahe-export');
const aheExported = await runCli([
const aheExported = await runCliInProcess([
'ahe',
'export',
'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
const taskRunId = `long-task-run-${'x'.repeat(320)}`;
await writeFile(specPath, JSON.stringify(spec), 'utf8');

const run = await runCli([
const run = await runCliInProcess([
'task',
'run',
specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
]);
assert.equal(run.code, 0, run.stderr);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

const resumed = await runCli([
const resumed = await runCliInProcess([
'task',
'resume',
taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
assert.match(resumed.stdout, /resumed: parked-run/);
assert.match(resumed.stdout, /status: completed/);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
);

const outDir = join(dir, 'out');
const result = await runCli([
const result = await runCliInProcess([
'task',
'retry-failed',
priorPath,
Expand Down
77 changes: 41 additions & 36 deletions packages/headless/src/__tests__/contamination-scan-cli.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
scheduledCellLogPath,
trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
}
}

/**
* The script's exported entry, invoked in process with the executable
* footer's contract mirrored exactly: a thrown error lands on stderr and
* exits 2. The real-subprocess route stays in 'refuses an invocation it
* cannot act on' as the representative coverage of that footer itself.
*/
async function runScanScript(
argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
const { main } = (await import(
new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
)) as { main: (argv?: string[]) => Promise<number> };
const {
result: code,
stdout,
stderr,
} = await withCapturedProcessIo(async () => {
try {
return await main(argv);
} catch (error) {
process.stderr.write(
`${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
);
return 2;
}
});
return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
const jsonPath = join(runRoot, 'report.json');
let code = 0;
try {
await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
} catch (error) {
code = (error as { code?: number }).code ?? -1;
}
const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
[{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
async (runRoot, armIds) => {
const markdownPath = join(runRoot, 'report.md');
await execFileAsync(process.execPath, [
SCRIPT,
'--run-root',
runRoot,
'--markdown',
markdownPath,
]);
const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
assert.equal(code, 0);
assert.match(
await readFile(markdownPath, 'utf8'),
new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
});

test('refuses a flag it does not know', async () => {
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
(error: { code?: number }) => {
assert.equal(error.code, 2);
return true;
},
);
const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
assert.equal(code, 2);
});

// A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
test('says what is missing when a run root is not one', async () => {
const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
try {
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
(error: { code?: number; stderr?: string }) => {
assert.equal(error.code, 2);
assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
return true;
},
);
const { code, stderr } = await runScanScript(['--run-root', empty]);
assert.equal(code, 2);
assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
} finally {
await rm(empty, { recursive: true, force: true });
}
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
try {
await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
(error: { code?: number; stderr?: string }) => {
assert.equal(error.code, 2);
assert.match(error.stderr ?? '', /names no cells/);
return true;
},
);
const { code, stderr } = await runScanScript(['--run-root', runRoot]);
assert.equal(code, 2);
assert.match(stderr, /names no cells/);
} finally {
await rm(runRoot, { recursive: true, force: true });
}
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
test(headless): run ordinary CLI command semantics in process#2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

UncertaintyDeterminesYou4ndMe
wants to merge
1
commit into

from

 Closed


test(headless): run ordinary CLI command semantics in process
#2479
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


83 changes: 61 additions & 22 deletions
packages/headless/src/__tests__/cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
 return new Promise((resolve) => {
 const child = spawn(process.execPath, [cliPath, ...args], {
 env: { ...process.env, ...options.env },
 env: { ...process.env },
 });
 let stdout = '';
 let stderr = '';
Expand All@@ -35,6 +34,46 @@ function runCli(
 });
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
 args: string[],
 options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
 const mapped = mapLegacyMakaHeadlessArgs(args);
 assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
 const envOverrides = Object.entries(options.env ?? {});
 const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
 for (const [key, value] of envOverrides) process.env[key] = value;
 const savedExitCode = process.exitCode;
 try {
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await runMakaEvalCli(mapped);
 } catch (error) {
 // Mirror the bin's fatal handler: report the error and fail with 1.
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 1;
 }
 });
 return { code, stdout, stderr };
 } finally {
 process.exitCode = savedExitCode;
 for (const [key, value] of savedEnv) {
 if (value === undefined) delete process.env[key];
 else process.env[key] = value;
 }
 }
}

describe('maka-headless CLI', () => {
 test('task-run readers report an actionable error for a non-Headless root', async () => {
 const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand DownExpand Up@@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 1);
 assert.match(result.stderr, /protectedPaths/);
 } finally {
Expand All@@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const result = await runCli([
 const result = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
 const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
 try {
 await mkdir(join(dir, 'fixture'), { recursive: true });
 const missingUrl = await runCli([
 const missingUrl = await runCliInProcess([
 'harbor',
 'run',
 '--backend',
Expand All@@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
 assert.equal(missingUrl.code, 1);
 assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

 const missingToken = await runCli(
 const missingToken = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
 await mkdir(fixture, { recursive: true });
 await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

 const result = await runCli(
 const result = await runCliInProcess(
 [
 'harbor',
 'run',
Expand DownExpand Up@@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
 };
 const specPath = join(dir, 'spec.json');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');
 const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
 const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
 assert.equal(result.code, 0, result.stderr);
 const records = await readResults(join(dir, 'out', 'results.jsonl'));
 assert.equal(records[0]?.passed, false);
Expand DownExpand Up@@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
 const outDir = join(dir, 'out');
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
 assert.equal(run.code, 1);
 assert.match(run.stdout, /taskRunId: task-run-1/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
 assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
 assert.ok(Array.isArray(inspectDocument.attempts));

 const humanInspect = await runCli([
 const humanInspect = await runCliInProcess([
 'task',
 'inspect',
 'task-run-1',
Expand All@@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
 assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

 const exportDir = join(dir, 'manual-export');
 const exported = await runCli([
 const exported = await runCliInProcess([
 'task',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
 }

 const aheExportDir = join(dir, 'ahe-export');
 const aheExported = await runCli([
 const aheExported = await runCliInProcess([
 'ahe',
 'export',
 'task-run-1',
Expand DownExpand Up@@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
 const taskRunId = `long-task-run-${'x'.repeat(320)}`;
 await writeFile(specPath, JSON.stringify(spec), 'utf8');

 const run = await runCli([
 const run = await runCliInProcess([
 'task',
 'run',
 specPath,
Expand All@@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
 ]);
 assert.equal(run.code, 0, run.stderr);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
 const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
 for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

 const resumed = await runCli([
 const resumed = await runCliInProcess([
 'task',
 'resume',
 taskRunId,
Expand All@@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
 assert.match(resumed.stdout, /resumed: parked-run/);
 assert.match(resumed.stdout, /status: completed/);

 const inspect = await runCli([
 const inspect = await runCliInProcess([
 'task',
 'inspect',
 taskRunId,
Expand DownExpand Up@@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
 );

 const outDir = join(dir, 'out');
 const result = await runCli([
 const result = await runCliInProcess([
 'task',
 'retry-failed',
 priorPath,
Expand Down

77 changes: 41 additions & 36 deletions
packages/headless/src/__tests__/contamination-scan-cli.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ import {
 scheduledCellLogPath,
 trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand DownExpand Up@@ -123,14 +124,38 @@ async function withRunRoot<T>(
 }
}

/**
 * The script's exported entry, invoked in process with the executable
 * footer's contract mirrored exactly: a thrown error lands on stderr and
 * exits 2. The real-subprocess route stays in 'refuses an invocation it
 * cannot act on' as the representative coverage of that footer itself.
 */
async function runScanScript(
 argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
 const { main } = (await import(
 new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
 )) as { main: (argv?: string[]) => Promise<number> };
 const {
 result: code,
 stdout,
 stderr,
 } = await withCapturedProcessIo(async () => {
 try {
 return await main(argv);
 } catch (error) {
 process.stderr.write(
 `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
 );
 return 2;
 }
 });
 return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
 const jsonPath = join(runRoot, 'report.json');
 let code = 0;
 try {
 await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
 } catch (error) {
 code = (error as { code?: number }).code ?? -1;
 }
 const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
 return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand DownExpand Up@@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
 [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
 async (runRoot, armIds) => {
 const markdownPath = join(runRoot, 'report.md');
 await execFileAsync(process.execPath, [
 SCRIPT,
 '--run-root',
 runRoot,
 '--markdown',
 markdownPath,
 ]);
 const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
 assert.equal(code, 0);
 assert.match(
 await readFile(markdownPath, 'utf8'),
 new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All@@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
 });

 test('refuses a flag it does not know', async () => {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
 (error: { code?: number }) => {
 assert.equal(error.code, 2);
 return true;
 },
 );
 const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
 assert.equal(code, 2);
 });

 // A cell whose trajectory never landed was not searched, and a zero exit here
Expand DownExpand Up@@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
 test('says what is missing when a run root is not one', async () => {
 const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
 try {
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', empty]);
 assert.equal(code, 2);
 assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
 } finally {
 await rm(empty, { recursive: true, force: true });
 }
Expand DownExpand Up@@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
 const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
 try {
 await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
 await assert.rejects(
 execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
 (error: { code?: number; stderr?: string }) => {
 assert.equal(error.code, 2);
 assert.match(error.stderr ?? '', /names no cells/);
 return true;
 },
 );
 const { code, stderr } = await runScanScript(['--run-root', runRoot]);
 assert.equal(code, 2);
 assert.match(stderr, /names no cells/);
 } finally {
 await rm(runRoot, { recursive: true, force: true });
 }
Expand Down
Loading