feat(project): add project add evaluator code-based

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+ judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+ output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+ # the extra owns the autoevals version, so pinning it here only conflicts.
+ "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+ "litellm>=1.60,<1.85",
+{{else}}
+ "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+ behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+ judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+ supply those when you invoke the evaluator, or the metric returns
+ `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["bedrock:InvokeModel"],
+ "Resource": "*"
+ }
+ ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ EvaluatorInput,
+ EvaluatorOutput,
+ custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+ return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+ # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+ # the extra owns the deepeval version, so pinning it here only conflicts.
+ "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+ # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+ # allows the botocore that bedrock-agentcore requires.
+ "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+ handler receives an `EvaluatorInput` (the session / trace / tool-call to
+ grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+ ships as a stub that returns `Pass` for everything — replace the `TODO` with
+ your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+ [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+ Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ # inspect input.session_spans / input.target_trace_id / input.target_span_id
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+ }
+ ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+ custom_code_based_evaluator,
+ EvaluatorInput,
+ EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+ """Evaluate agent behavior with custom logic.
+
+ Args:
+ input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+ Returns:
+ EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+ """
+ # TODO: Replace with your evaluation logic
+ return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+ "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
  }
  }
 }
+
+export function toPythonPackageName(name: string): string {
+ return name
+ .replace(/[^a-zA-Z0-9._-]/g, "-")
+ .replace(/^[^a-zA-Z0-9]+/, "")
+ .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
  break;
  }
  case "evaluator": {
- projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ if (input.scaffold) {
+ yield { message: "Scaffolding evaluator in project" };
+ const outputPath = join(project.rootPath, "app", input.scaffold.name);
+ if (existsSync(outputPath))
+ throw new InputValidationError(
+ `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+ );
+ scaffoldedPaths.push(outputPath);
+ const result = await getEvaluatorTemplateResolver({
+ assetSource: this.assetSource,
+ templateRenderer: this.templateRenderer,
+ }).resolve(input.scaffold);
+ await result.tree.write(dirname(outputPath));
+ projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+ } else {
+ projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+ }
  break;
  }
  case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+ EvaluatorLibrary,
+ ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+ EvaluatorLibrary,
+ { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+ deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+ autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+ const timeoutSeconds =
+ input.timeoutSeconds ??
+ (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+ return {
+ name: input.name,
+ level: input.level,
+ ...(input.description && { description: input.description }),
+ config: {
+ codeBased: {
+ managed: {
+ codeLocation: `app/${input.name}`,
+ entrypoint: "lambda_function.handler",
+ timeoutSeconds,
+ additionalPolicies: ["execution-role-policy.json"],
+ },
+ },
+ },
+ ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+ ...(input.tags && { tags: input.tags }),
+ };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+ const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+ if (input.metric) {
+ context["EvaluatorClass"] = input.metric.metricClass;
+ context["Model"] = input.model ?? "";
+ context["ModelProviderBedrock"] = input.model !== undefined;
+ context["EvaluatorParams"] = "";
+ }
+ return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+ assetSource: AssetSource;
+ templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+ config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+ return {
+ async resolve(input) {
+ const assetDir = input.metric
+ ? EVALUATOR_ASSETS[input.metric.library].assetDir
+ : EMPTY_ASSET_DIR;
+ const tree = await FsTreeNode.fromAssetSource(
+ { assetSource: config.assetSource },
+ { assetDir },
+ {
+ rootDirName: input.name,
+ transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+ },
+ );
+ return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+ },
+ };
+}