Skip to content

feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
notgitika merged 4 commits into from
May 12, 2026
Merged

feat(evaluator): add kmsKeyArn support for custom evaluator #994

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7b599b0
feat(evaluator): Add kmsKeyArn support for custom evaluator
aws-aditya21 Apr 27, 2026
6b3c21d
fix: sync package-lock.json with package.json
notgitika May 12, 2026
64421a9
fix: revert unrelated dep bumps and fix formatting
notgitika May 12, 2026
c7302c0
fix: sync package-lock.json with updated dependencies
notgitika May 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
866 changes: 467 additions & 399 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 8 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
"@aws-sdk/client-bedrock": "^3.1012.0",
"@aws-sdk/client-bedrock-agent": "^3.1012.0",
"@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
"@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
"@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
"@aws-sdk/client-bedrock-runtime": "^3.893.0",
"@aws-sdk/client-cloudformation": "^3.893.0",
"@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
"lint-staged": "^16.2.7",
"node-pty": "^1.1.0",
"prettier": "^3.7.4",
"secretlint": "^13.0.0",
"secretlint": "^12.2.0",
"tsx": "^4.21.0",
"typescript": "^5",
"typescript-eslint": "^8.50.1",
"vitest": "^4.0.18"
},
"overridesComments": {
"minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
"glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
"glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
"fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
"@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
},
"overrides": {
"minimatch": "10.2.4",
"glob": "^13.0.0"
"glob": "^13.0.0",
"fast-xml-parser": "5.5.7",
"@aws-sdk/xml-builder": "3.972.15"
},
"engines": {
"node": ">=20"
Expand Down
2 changes: 2 additions & 0 deletions src/cli/aws/agentcore-control.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
llmAsAJudge?: GetEvaluatorLlmConfig;
codeBased?: GetEvaluatorCodeBasedConfig;
};
kmsKeyArn?: string;
tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
status: response.status ?? 'UNKNOWN',
description: response.description,
evaluatorConfig,
kmsKeyArn: response.kmsKeyArn,
tags,
};
}
Expand Down
43 changes: 43 additions & 0 deletions src/cli/commands/import/__tests__/import-evaluator.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

expect(result.tags).toBeUndefined();
});

it('forwards kmsKeyArn when present', () => {
const detail: GetEvaluatorResult = {
evaluatorId: 'eval-kms',
evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
evaluatorName: 'kms_eval',
level: 'SESSION',
status: 'ACTIVE',
evaluatorConfig: {
llmAsAJudge: {
model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
instructions: 'Evaluate',
ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
},
},
kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
};

const result = toEvaluatorSpec(detail, 'kms_eval');

expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
});

it('omits kmsKeyArn when not present', () => {
const detail: GetEvaluatorResult = {
evaluatorId: 'eval-no-kms',
evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
evaluatorName: 'no_kms_eval',
level: 'SESSION',
status: 'ACTIVE',
evaluatorConfig: {
llmAsAJudge: {
model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
instructions: 'Evaluate',
ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
},
},
};

const result = toEvaluatorSpec(detail, 'no_kms_eval');

expect(result.kmsKeyArn).toBeUndefined();
});
});

// ============================================================================
Expand Down
1 change: 1 addition & 0 deletions src/cli/commands/import/import-evaluator.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
level,
...(detail.description && { description: detail.description }),
config,
...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
};
}
Expand Down
13 changes: 12 additions & 1 deletion src/cli/primitives/EvaluatorPrimitive.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
level: EvaluationLevel;
description?: string;
config: EvaluatorConfig;
kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
'--config <path>',
'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
)
.option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
.option('--json', 'Output as JSON [non-interactive]')
.action(
async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
lambdaArn?: string;
timeout?: string;
config?: string;
kmsKeyArn?: string;
json?: boolean;
}) => {
if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
};
}

if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
fail(
'--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
);
}

const result = await this.add({
name: cliOptions.name!,
level: levelResult.data!,
config: configJson,
kmsKeyArn: cliOptions.kmsKeyArn,
});

if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
level: options.level,
...(options.description && { description: options.description }),
config: options.config,
...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
};

project.evaluators.push(evaluator);
Expand Down
2 changes: 2 additions & 0 deletions src/cli/tui/hooks/useCreateEvaluator.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
name: string;
level: string;
config: EvaluatorConfig;
kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
name: config.name,
level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
config: config.config,
kmsKeyArn: config.kmsKeyArn,
})
);
if (!addResult.success) {
Expand Down
23 changes: 22 additions & 1 deletion src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
const isLambdaArnStep = wizard.step === 'lambda-arn';
const isTimeoutStep = wizard.step === 'timeout';
const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
const isConfirmStep = wizard.step === 'confirm';

const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

// Build confirm fields based on evaluator type
const confirmFields = useMemo(() => {
const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

if (wizard.evaluatorType === 'llm-as-a-judge') {
const llm = wizard.config.config.llmAsAJudge!;
return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
},
{ label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
...kmsField,
];
}

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
{ label: 'Code', value: managed.codeLocation },
{ label: 'Entrypoint', value: managed.entrypoint },
{ label: 'Timeout', value: `${managed.timeoutSeconds}s` },
...kmsField,
];
}

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
{ label: 'Name', value: wizard.config.name },
{ label: 'Level', value: wizard.config.level },
{ label: 'Lambda ARN', value: external.lambdaArn },
...kmsField,
];
}, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
/>
)}

{isKmsKeyArnStep && (
<TextInput
key="kms-key-arn"
prompt="KMS key ARN for encryption (optional, press Enter to skip)"
initialValue=""
onSubmit={wizard.setKmsKeyArn}
onCancel={() => wizard.goBack()}
customValidation={value =>
value === '' ||
isValidKmsKeyArn(value) ||
'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
}
/>
)}

{isConfirmStep && <ConfirmReview fields={confirmFields} />}
</Panel>
</Screen>
Expand Down
3 changes: 3 additions & 0 deletions src/cli/tui/screens/evaluator/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
| 'ratingScale-custom'
| 'lambda-arn'
| 'timeout'
| 'kms-key-arn'
| 'confirm';

export interface AddEvaluatorConfig {
name: string;
level: EvaluationLevel;
config: EvaluatorConfig;
kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
'ratingScale-custom': 'Scale',
'lambda-arn': 'Lambda',
timeout: 'Timeout',
'kms-key-arn': 'KMS Key',
confirm: 'Confirm',
};

Expand Down
21 changes: 20 additions & 1 deletion src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
'model',
'instructions',
'ratingScale',
'kms-key-arn',
'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
'name',
'level',
'timeout',
'kms-key-arn',
'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
'name',
'level',
'lambda-arn',
'kms-key-arn',
'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
const [lambdaArn, setLambdaArnState] = useState('');
const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
const [kmsKeyArn, setKmsKeyArnState] = useState('');
const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

// Build the final config based on current state
const config: AddEvaluatorConfig = useMemo(() => {
const kms = kmsKeyArn || undefined;
if (evaluatorType === 'llm-as-a-judge') {
return {
name,
level,
config: { llmAsAJudge: llmConfig },
...(kms && { kmsKeyArn: kms }),
};
}

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
external: { lambdaArn },
},
},
...(kms && { kmsKeyArn: kms }),
};
}

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
},
},
},
...(kms && { kmsKeyArn: kms }),
};
}, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
}, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
[nextStep]
);

const setKmsKeyArn = useCallback(
(arn: string) => {
setKmsKeyArnState(arn);
const next = nextStep('kms-key-arn');
if (next) setStep(next);
},
[nextStep]
);

const reset = useCallback(() => {
setEvaluatorType('code-based');
setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
setLambdaArnState('');
setTimeoutState(DEFAULT_CODE_TIMEOUT);
setKmsKeyArnState('');
setStep('evaluator-type');
}, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
setCustomRatingScale,
setLambdaArn,
setTimeout,
setKmsKeyArn,
reset,
};
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
feat(evaluator): add kmsKeyArn support for custom evaluator by aws-aditya21 · Pull Request #994 · aws/agentcore-cli · GitHub
Skip to content
feat(evaluator): add kmsKeyArn support for custom evaluator#994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

notgitika
merged 4 commits into

from

May 12, 2026
 Merged


feat(evaluator): add kmsKeyArn support for custom evaluator
#994
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
7b599b0

feat(evaluator): Add kmsKeyArn support for custom evaluator


aws-aditya21 Apr 27, 2026
6b3c21d

fix: sync package-lock.json with package.json


notgitika May 12, 2026
64421a9

fix: revert unrelated dep bumps and fix formatting


notgitika May 12, 2026
c7302c0

fix: sync package-lock.json with updated dependencies


notgitika May 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


866 changes: 467 additions & 399 deletions
package-lock.json

View file


Open in desktop


Large diffs are not rendered by default.


12 changes: 8 additions & 4 deletions
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -78,7 +78,7 @@
 "@aws-sdk/client-bedrock": "^3.1012.0",
 "@aws-sdk/client-bedrock-agent": "^3.1012.0",
 "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
 "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
 "@aws-sdk/client-bedrock-runtime": "^3.893.0",
 "@aws-sdk/client-cloudformation": "^3.893.0",
 "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
Expand DownExpand Up@@ -141,19 +141,23 @@
 "lint-staged": "^16.2.7",
 "node-pty": "^1.1.0",
 "prettier": "^3.7.4",
 "secretlint": "^13.0.0",
 "secretlint": "^12.2.0",
 "tsx": "^4.21.0",
 "typescript": "^5",
 "typescript-eslint": "^8.50.1",
 "vitest": "^4.0.18"
 },
 "overridesComments": {
 "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
 "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
 "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
 "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
 },
 "overrides": {
 "minimatch": "10.2.4",
 "glob": "^13.0.0"
 "glob": "^13.0.0",
 "fast-xml-parser": "5.5.7",
 "@aws-sdk/xml-builder": "3.972.15"
 },
 "engines": {
 "node": ">=20"
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/aws/agentcore-control.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
 llmAsAJudge?: GetEvaluatorLlmConfig;
 codeBased?: GetEvaluatorCodeBasedConfig;
 };
 kmsKeyArn?: string;
 tags?: Record<string, string>;
}

Expand DownExpand Up@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
 status: response.status ?? 'UNKNOWN',
 description: response.description,
 evaluatorConfig,
 kmsKeyArn: response.kmsKeyArn,
 tags,
 };
}
Expand Down

43 changes: 43 additions & 0 deletions
src/cli/commands/import/__tests__/import-evaluator.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {

 expect(result.tags).toBeUndefined();
 });

 it('forwards kmsKeyArn when present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
 evaluatorName: 'kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
 };

 const result = toEvaluatorSpec(detail, 'kms_eval');

 expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
 });

 it('omits kmsKeyArn when not present', () => {
 const detail: GetEvaluatorResult = {
 evaluatorId: 'eval-no-kms',
 evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
 evaluatorName: 'no_kms_eval',
 level: 'SESSION',
 status: 'ACTIVE',
 evaluatorConfig: {
 llmAsAJudge: {
 model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
 instructions: 'Evaluate',
 ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
 },
 },
 };

 const result = toEvaluatorSpec(detail, 'no_kms_eval');

 expect(result.kmsKeyArn).toBeUndefined();
 });
});

// ============================================================================
Expand Down

1 change: 1 addition & 0 deletions
src/cli/commands/import/import-evaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
 level,
 ...(detail.description && { description: detail.description }),
 config,
 ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
 ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
 };
}
Expand Down

13 changes: 12 additions & 1 deletion
src/cli/primitives/EvaluatorPrimitive.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { findConfigRoot } from '../../lib';
import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
import { getErrorMessage } from '../errors';
import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
import { runCliCommand } from '../telemetry/cli-command-run.js';
Expand All@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
 level: EvaluationLevel;
 description?: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export type RemovableEvaluator = RemovableResource;
Expand DownExpand Up@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 '--config <path>',
 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
 )
 .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
 .option('--json', 'Output as JSON [non-interactive]')
 .action(
 async (cliOptions: {
Expand All@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 lambdaArn?: string;
 timeout?: string;
 config?: string;
 kmsKeyArn?: string;
 json?: boolean;
 }) => {
 if (!findConfigRoot()) {
Expand DownExpand Up@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 };
 }

 if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
 fail(
 '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 );
 }

 const result = await this.add({
 name: cliOptions.name!,
 level: levelResult.data!,
 config: configJson,
 kmsKeyArn: cliOptions.kmsKeyArn,
 });

 if (!result.success) {
Expand DownExpand Up@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 level: options.level,
 ...(options.description && { description: options.description }),
 config: options.config,
 ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
 };

 project.evaluators.push(evaluator);
Expand Down

2 changes: 2 additions & 0 deletions
src/cli/tui/hooks/useCreateEvaluator.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
 name: string;
 level: string;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export function useCreateEvaluator() {
Expand All@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
 name: config.name,
 level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
 config: config.config,
 kmsKeyArn: config.kmsKeyArn,
 })
 );
 if (!addResult.success) {
Expand Down

23 changes: 22 additions & 1 deletion
src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
import type { SelectableItem } from '../../components';
import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
import { HELP_TEXT } from '../../constants';
Expand DownExpand Up@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
 const isLambdaArnStep = wizard.step === 'lambda-arn';
 const isTimeoutStep = wizard.step === 'timeout';
 const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
 const isConfirmStep = wizard.step === 'confirm';

 const evaluatorTypeNav = useListNavigation({
Expand DownExpand Up@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames

 // Build confirm fields based on evaluator type
 const confirmFields = useMemo(() => {
 const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];

 if (wizard.evaluatorType === 'llm-as-a-judge') {
 const llm = wizard.config.config.llmAsAJudge!;
 return [
Expand All@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
 },
 { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
 ...kmsField,
 ];
 }

Expand All@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Code', value: managed.codeLocation },
 { label: 'Entrypoint', value: managed.entrypoint },
 { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
 ...kmsField,
 ];
 }

Expand All@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 { label: 'Name', value: wizard.config.name },
 { label: 'Level', value: wizard.config.level },
 { label: 'Lambda ARN', value: external.lambdaArn },
 ...kmsField,
 ];
 }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);

Expand DownExpand Up@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 />
 )}

 {isKmsKeyArnStep && (
 <TextInput
 key="kms-key-arn"
 prompt="KMS key ARN for encryption (optional, press Enter to skip)"
 initialValue=""
 onSubmit={wizard.setKmsKeyArn}
 onCancel={() => wizard.goBack()}
 customValidation={value =>
 value === '' ||
 isValidKmsKeyArn(value) ||
 'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
 }
 />
 )}

 {isConfirmStep && <ConfirmReview fields={confirmFields} />}
 </Panel>
 </Screen>
Expand Down

3 changes: 3 additions & 0 deletions
src/cli/tui/screens/evaluator/types.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
 | 'ratingScale-custom'
 | 'lambda-arn'
 | 'timeout'
 | 'kms-key-arn'
 | 'confirm';

export interface AddEvaluatorConfig {
 name: string;
 level: EvaluationLevel;
 config: EvaluatorConfig;
 kmsKeyArn?: string;
}

export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
Expand All@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
 'ratingScale-custom': 'Scale',
 'lambda-arn': 'Lambda',
 timeout: 'Timeout',
 'kms-key-arn': 'KMS Key',
 confirm: 'Confirm',
};

Expand Down

21 changes: 20 additions & 1 deletion
src/cli/tui/screens/evaluator/useAddEvaluatorWizard.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
 'model',
 'instructions',
 'ratingScale',
 'kms-key-arn',
 'confirm',
];
const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
Expand All@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'timeout',
 'kms-key-arn',
 'confirm',
];
const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
Expand All@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
 'name',
 'level',
 'lambda-arn',
 'kms-key-arn',
 'confirm',
];

Expand DownExpand Up@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
 const [lambdaArn, setLambdaArnState] = useState('');
 const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
 const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
 const [kmsKeyArn, setKmsKeyArnState] = useState('');
 const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');

 const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
Expand DownExpand Up@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {

 // Build the final config based on current state
 const config: AddEvaluatorConfig = useMemo(() => {
 const kms = kmsKeyArn || undefined;
 if (evaluatorType === 'llm-as-a-judge') {
 return {
 name,
 level,
 config: { llmAsAJudge: llmConfig },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
 external: { lambdaArn },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }

Expand All@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
 },
 },
 },
 ...(kms && { kmsKeyArn: kms }),
 };
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
 }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);

 const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
 setEvaluatorType(type);
Expand DownExpand Up@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
 [nextStep]
 );

 const setKmsKeyArn = useCallback(
 (arn: string) => {
 setKmsKeyArnState(arn);
 const next = nextStep('kms-key-arn');
 if (next) setStep(next);
 },
 [nextStep]
 );

 const reset = useCallback(() => {
 setEvaluatorType('code-based');
 setCodeBasedType('managed');
Expand All@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
 setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
 setLambdaArnState('');
 setTimeoutState(DEFAULT_CODE_TIMEOUT);
 setKmsKeyArnState('');
 setStep('evaluator-type');
 }, []);

Expand All@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
 setCustomRatingScale,
 setLambdaArn,
 setTimeout,
 setKmsKeyArn,
 reset,
 };
}
Loading
Loading