feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hweinstock merged 1 commit into from
May 11, 2026
Merged

feat: wire telemetry into all remove.* commands #1069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions integ-tests/add-remove-ab-test.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
readProjectConfig,
runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
const result = await runCLI(args, cwd);
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

afterAll(async () => {
await project.cleanup();
telemetry.destroy();
});

it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
});

it('removes ab-test', async () => {
const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
env: telemetry.env,
});
expect(result.exitCode).toBe(0);
const json = JSON.parse(result.stdout);
expect(json.success).toBe(true);

// Verify removal from agentcore.json
const spec = await readProjectConfig(project.projectPath);
const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
expect(abTest).toBeUndefined();
telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
});

it('remove returns error for non-existent test', async () => {
const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
env: telemetry.env,
});
expect(result.exitCode).toBe(1);
const json = JSON.parse(result.stdout);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
});
});
46 changes: 29 additions & 17 deletions integ-tests/add-remove-config-bundle.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
runFailure,
runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

afterAll(async () => {
await project.cleanup();
telemetry.destroy();
});

// ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
expect(json.bundleName).toBe('InlineBundle');

const config = await readProjectConfig(project.projectPath);
const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
expect(bundle).toBeDefined();
expect(bundle!.type).toBe('ConfigurationBundle');
expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
expect(json.bundleName).toBe('FileBundle');

const config = await readProjectConfig(project.projectPath);
const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
const bundle = config.configBundles.find(b => b.name === 'FileBundle');
expect(bundle).toBeDefined();
expect(Object.keys(bundle!.components)).toHaveLength(2);
});
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
expect(json.bundleName).toBe('FullOptsBundle');

const config = await readProjectConfig(project.projectPath);
const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
expect(bundle).toBeDefined();
expect(bundle!.description).toBe('A bundle with all optional fields');
expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
expect(json.bundleName).toBe('PlaceholderBundle');

const config = await readProjectConfig(project.projectPath);
const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
expect(bundle).toBeDefined();
const keys = Object.keys(bundle!.components);
expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

describe('remove config-bundle', () => {
it('removes an existing config bundle', async () => {
const json = await runSuccess(
const result = await runCLI(
['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
project.projectPath
project.projectPath,
{ env: telemetry.env }
);

expect(result.exitCode).toBe(0);
const json = JSON.parse(result.stdout);
expect(json.success).toBe(true);

const config = await readProjectConfig(project.projectPath);
const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
expect(bundle).toBeUndefined();
telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
});

it('returns error for non-existent bundle', async () => {
const json = await runFailure(
const result = await runCLI(
['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
project.projectPath
project.projectPath,
{ env: telemetry.env }
);

expect(result.exitCode).toBe(1);
const json = JSON.parse(result.stdout);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
});

it('removes all remaining config bundles one by one', async () => {
const configBefore = await readProjectConfig(project.projectPath);
const remaining = configBefore.configBundles!.map(b => b.name);
const remaining = configBefore.configBundles.map(b => b.name);

for (const name of remaining) {
await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
}

const configAfter = await readProjectConfig(project.projectPath);
expect(configAfter.configBundles!).toHaveLength(0);
expect(configAfter.configBundles).toHaveLength(0);
});
});

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
}

const config = await readProjectConfig(project.projectPath);
expect(config.configBundles!).toHaveLength(bundleNames.length);
expect(config.configBundles).toHaveLength(bundleNames.length);

for (const name of bundleNames) {
expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
expect(config.configBundles.find(b => b.name === name)).toBeDefined();
}
});

it('removing one bundle does not affect others', async () => {
await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

const config = await readProjectConfig(project.projectPath);
expect(config.configBundles!).toHaveLength(2);
expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
expect(config.configBundles).toHaveLength(2);
expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
});

afterAll(async () => {
Expand Down
12 changes: 10 additions & 2 deletions integ-tests/add-remove-evaluator.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
const result = await runCLI(args, cwd);
const result = await runCLI(args, cwd, { env: telemetry.env });
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
const json: unknown = parseJsonOutput(result.stdout);
expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
const result = await runCLI(args, cwd);
const result = await runCLI(args, cwd, { env: telemetry.env });
expect(result.exitCode).toBe(1);
const json: unknown = parseJsonOutput(result.stdout);
expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

afterAll(async () => {
await project.cleanup();
telemetry.destroy();
});

describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

const config = await readProjectConfig(project.projectPath);
expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
});

it('removes the evaluator after online eval is gone', async () => {
await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

const config = await readProjectConfig(project.projectPath);
expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
});
});

describe('error cases', () => {
it('fails to remove non-existent evaluator', async () => {
const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
});

it('fails to remove non-existent online eval config', async () => {
const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
});

it('rejects evaluator with missing --level', async () => {
Expand Down
32 changes: 30 additions & 2 deletions integ-tests/add-remove-gateway.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

afterAll(async () => {
await project.cleanup();
telemetry.destroy();
});

describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
});

it('removes the gateway target', async () => {
const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
env: telemetry.env,
});

expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
const targets = gateway?.targets ?? [];
const found = targets.find((t: { name: string }) => t.name === targetName);
expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
});

it('removes the gateway', async () => {
const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
env: telemetry.env,
});

expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
const gateways = mcpSpec.agentCoreGateways ?? [];
const found = gateways.find((g: { name: string }) => g.name === gatewayName);
expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
});

it('fails to remove non-existent gateway', async () => {
const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
env: telemetry.env,
});
expect(result.exitCode).toBe(1);
telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
});

it('fails to remove non-existent gateway target', async () => {
const result = await runCLI(
['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
project.projectPath,
{ env: telemetry.env }
);
expect(result.exitCode).toBe(1);
telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
});
});
});
Expand Down
17 changes: 14 additions & 3 deletions integ-tests/add-remove-policy.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

afterAll(async () => {
await project.cleanup();
telemetry.destroy();
});

describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
it('removes a policy with --engine flag', async () => {
const result = await runCLI(
['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
project.projectPath
project.projectPath,
{ env: telemetry.env }
);

expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
const policy = engine!.policies.find(p => p.name === policyName);
expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
});
});

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
});

it('fails to remove non-existent policy', async () => {
const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
env: telemetry.env,
});

expect(result.exitCode).toBe(1);
const json = JSON.parse(result.stdout);
expect(json.success).toBe(false);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
});

it('fails to remove non-existent policy engine', async () => {
const result = await runCLI(
['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
project.projectPath
project.projectPath,
{ env: telemetry.env }
);

expect(result.exitCode).toBe(1);
const json = JSON.parse(result.stdout);
expect(json.success).toBe(false);
expect(json.error).toContain('not found');
telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
});

it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
feat: wire telemetry into all remove.* commands#1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

Hweinstock
merged 1 commit into

from

May 11, 2026
 Merged


feat: wire telemetry into all remove.* commands
#1069
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 16 additions & 3 deletions
integ-tests/add-remove-ab-test.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,8 +5,11 @@ import {
 readProjectConfig,
 runCLI,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand DownExpand Up@@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 it('requires --name for JSON mode', async () => {
Expand DownExpand Up@@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => {
 });

 it('removes ab-test', async () => {
 const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 // Verify removal from agentcore.json
 const spec = await readProjectConfig(project.projectPath);
 const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest');
 expect(abTest).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' });
 });

 it('remove returns error for non-existent test', async () => {
 const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' });
 });
});

46 changes: 29 additions & 17 deletions
integ-tests/add-remove-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,10 +7,13 @@ import {
 runFailure,
 runSuccess,
} from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove config-bundle', () => {
 let project: TestProject;

Expand All@@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 // ── Add lifecycle ─────────────────────────────────────────────────────
Expand All@@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('InlineBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.type).toBe('ConfigurationBundle');
 expect(bundle!.branchName).toBe('mainline');
Expand DownExpand Up@@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FileBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FileBundle');
 const bundle = config.configBundles.find(b => b.name === 'FileBundle');
 expect(bundle).toBeDefined();
 expect(Object.keys(bundle!.components)).toHaveLength(2);
 });
Expand DownExpand Up@@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('FullOptsBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle');
 const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle');
 expect(bundle).toBeDefined();
 expect(bundle!.description).toBe('A bundle with all optional fields');
 expect(bundle!.branchName).toBe('feature-branch');
Expand All@@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => {
 expect(json.bundleName).toBe('PlaceholderBundle');

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle');
 const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle');
 expect(bundle).toBeDefined();
 const keys = Object.keys(bundle!.components);
 expect(keys).toContain('{{runtime:AgentA}}');
Expand DownExpand Up@@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => {

 describe('remove config-bundle', () => {
 it('removes an existing config bundle', async () => {
 const json = await runSuccess(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(0);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(true);

 const config = await readProjectConfig(project.projectPath);
 const bundle = config.configBundles!.find(b => b.name === 'InlineBundle');
 const bundle = config.configBundles.find(b => b.name === 'InlineBundle');
 expect(bundle).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' });
 });

 it('returns error for non-existent bundle', async () => {
 const json = await runFailure(
 const result = await runCLI(
 ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' });
 });

 it('removes all remaining config bundles one by one', async () => {
 const configBefore = await readProjectConfig(project.projectPath);
 const remaining = configBefore.configBundles!.map(b => b.name);
 const remaining = configBefore.configBundles.map(b => b.name);

 for (const name of remaining) {
 await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath);
 }

 const configAfter = await readProjectConfig(project.projectPath);
 expect(configAfter.configBundles!).toHaveLength(0);
 expect(configAfter.configBundles).toHaveLength(0);
 });
 });

Expand All@@ -282,21 +294,21 @@ describe('integration: add and remove config-bundle', () => {
 }

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(bundleNames.length);
 expect(config.configBundles).toHaveLength(bundleNames.length);

 for (const name of bundleNames) {
 expect(config.configBundles!.find(b => b.name === name)).toBeDefined();
 expect(config.configBundles.find(b => b.name === name)).toBeDefined();
 }
 });

 it('removing one bundle does not affect others', async () => {
 await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.configBundles!).toHaveLength(2);
 expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined();
 expect(config.configBundles).toHaveLength(2);
 expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined();
 expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined();
 });

 afterAll(async () => {
Expand Down

12 changes: 10 additions & 2 deletions
integ-tests/add-remove-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
async function runSuccess(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', true);
Expand All@@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) {

/** Run a CLI command and assert it fails, returning parsed JSON output. */
async function runFailure(args: string[], cwd: string) {
 const result = await runCLI(args, cwd);
 const result = await runCLI(args, cwd, { env: telemetry.env });
 expect(result.exitCode).toBe(1);
 const json: unknown = parseJsonOutput(result.stdout);
 expect(json).toHaveProperty('success', false);
Expand All@@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('evaluator and online eval lifecycle', () => {
Expand DownExpand Up@@ -123,25 +127,29 @@ describe('integration: add and remove evaluators and online eval configs', () =>

 const config = await readProjectConfig(project.projectPath);
 expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
 });

 it('removes the evaluator after online eval is gone', async () => {
 await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);

 const config = await readProjectConfig(project.projectPath);
 expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
 });
 });

 describe('error cases', () => {
 it('fails to remove non-existent evaluator', async () => {
 const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
 });

 it('fails to remove non-existent online eval config', async () => {
 const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
 });

 it('rejects evaluator with missing --level', async () => {
Expand Down

32 changes: 30 additions & 2 deletions
integ-tests/add-remove-gateway.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
import { createTestProject, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

async function readProjectConfig(projectPath: string) {
 return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8'));
}
Expand All@@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('gateway lifecycle', () => {
Expand DownExpand Up@@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => {
 });

 it('removes the gateway target', async () => {
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const targets = gateway?.targets ?? [];
 const found = targets.find((t: { name: string }) => t.name === targetName);
 expect(found, `Target "${targetName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' });
 });

 it('removes the gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath);
 const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
 const json = JSON.parse(result.stdout);
Expand All@@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => {
 const gateways = mcpSpec.agentCoreGateways ?? [];
 const found = gateways.find((g: { name: string }) => g.name === gatewayName);
 expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy();
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' });
 });

 it('fails to remove non-existent gateway', async () => {
 const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, {
 env: telemetry.env,
 });
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' });
 });

 it('fails to remove non-existent gateway target', async () => {
 const result = await runCLI(
 ['remove', 'gateway-target', '--name', 'NonExistent', '--json'],
 project.projectPath,
 { env: telemetry.env }
 );
 expect(result.exitCode).toBe(1);
 telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' });
 });
 });
});
Expand Down

17 changes: 14 additions & 3 deletions
integ-tests/add-remove-policy.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
import type { TestProject } from '../src/test-utils/index.js';
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const telemetry = createTelemetryHelper();

describe('integration: add and remove policy engines and policies', () => {
 let project: TestProject;

Expand All@@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => {

 afterAll(async () => {
 await project.cleanup();
 telemetry.destroy();
 });

 describe('policy engine lifecycle', () => {
Expand DownExpand Up@@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => {
 it('removes a policy with --engine flag', async () => {
 const result = await runCLI(
 ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
Expand All@@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => {
 expect(engine, `Engine "${engineName}" should still exist`).toBeDefined();
 const policy = engine!.policies.find(p => p.name === policyName);
 expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined();
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' });
 });
 });

Expand DownExpand Up@@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => {
 });

 it('fails to remove non-existent policy', async () => {
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath);
 const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, {
 env: telemetry.env,
 });

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' });
 });

 it('fails to remove non-existent policy engine', async () => {
 const result = await runCLI(
 ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'],
 project.projectPath
 project.projectPath,
 { env: telemetry.env }
 );

 expect(result.exitCode).toBe(1);
 const json = JSON.parse(result.stdout);
 expect(json.success).toBe(false);
 expect(json.error).toContain('not found');
 telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' });
 });

 it('requires --engine when adding a policy', async () => {
Expand Down
Loading
Loading