feat(appkit): managed eval datasets + turn semantics (stack 3/5)

apps/dev-playground/config/agents/query/evals/dataset.eval.ts

@@ -0,0 +1,53 @@
+import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
+
+/**
+ * Dataset-driven eval: runs once per row of a Databricks managed evaluation
+ * dataset (a Unity Catalog table with `inputs`/`expectations` columns). The
+ * runner binds each row to `t.input`/`t.expected`.
+ *
+ * Run it (reading the dataset needs a workspace client + warehouse; judging the
+ * guidelines needs a judge model):
+ * appkit agent eval dataset --root apps/dev-playground --url http://localhost:8000 \
+ * --profile <profile> --warehouse <warehouse-id> --judge-model <endpoint>
+ *
+ * Row shape produced by the MLflow managed-dataset UI:
+ * inputs {"messages":[{"role":"user","content":"..."}]}
+ * expectations {"guidelines":{"value":["...","..."]}} (optional)
+ */
+
+/** Pull the last user message out of an MLflow `{messages:[...]}` input. */
+function userMessage(input: Record<string, unknown>): string {
+ const messages = Array.isArray(input.messages)
+ ? (input.messages as Array<{ role?: string; content?: string }>)
+ : [];
+ const last = [...messages].reverse().find((m) => m.role === "user");
+ return last?.content ?? "";
+}
+
+/** Read `expectations.guidelines` — the UI wraps the array as `{value: [...]}`. */
+function guidelines(expected: Record<string, unknown> | undefined): string[] {
+ const g = (expected?.guidelines as { value?: unknown } | undefined)?.value;
+ return Array.isArray(g) ? g.map(String) : [];
+}
+
+export default defineEval({
+ description: "Query agent satisfies each dataset row's guidelines",
+ // Point at your own managed evaluation dataset (catalog.schema.table).
+ dataset: { table: "main.mario.appkit_eval_dataset" },
+ async test(t) {
+ // One turn per row. For a multi-turn conversation, call `t.send` again
+ // (same thread); to start an independent turn in the same test, `t.reset()`.
+ await t.send(userMessage(t.input));
+ t.succeeded();
+
+ // Each guideline is judged against the reply — gate by default, so a miss
+ // fails the eval (chain `.soft()` to only track it). Skipped cleanly when no
+ // judge model is configured, so the eval still exercises the dataset read +
+ // drive path without one.
+ if (isJudgeConfigured()) {
+ for (const guideline of guidelines(t.expected)) {
+ (await t.judge.closedQA(guideline)).atLeast(0.5);
+ }
+ }
+ },
+});

apps/dev-playground/config/agents/query/evals/judge.eval.ts

@@ -0,0 +1,25 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * LLM-as-judge eval: scores the helper agent's weather answer with a judge
+ * model (via autoevals → a Databricks serving endpoint).
+ *
+ * Requires a judge model:
+ * appkit agent eval query --root apps/dev-playground --url http://localhost:8001 \
+ * --judge-model databricks-claude-sonnet-4-5
+ * (or set APPKIT_JUDGE_MODEL). Without it, t.judge.* errors with a clear message.
+ */
+export default defineEval({
+ description: "Helper weather answer is relevant (LLM judge)",
+ agent: "helper",
+ async test(t) {
+ await t.send("What's the weather in Brooklyn?");
+ t.succeeded();
+ // closedQA needs no ground truth — it judges the reply against a question.
+ (
+ await t.judge.closedQA(
+ "Does the response describe weather conditions for Brooklyn?",
+ )
+ ).atLeast(0.5);
+ },
+});

apps/dev-playground/config/agents/query/evals/smoke.eval.ts

@@ -0,0 +1,14 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * Example eval. The agent defaults to this directory's name (`query`).
+ * Run with:
+ * pnpm exec appkit agent eval query --root apps/dev-playground --url http://localhost:8000
+ */
+export default defineEval({
+ description: "Query dispatcher responds to a greeting",
+ async test(t) {
+ await t.send("Hi there!");
+ t.succeeded(); // gate: the turn completed
+ },
+});

apps/dev-playground/config/agents/query/evals/sum.eval.ts

@@ -0,0 +1,13 @@
+import { defineEval, includes } from "@databricks/appkit/beta";
+
+export default defineEval({
+ description: "Helper agent smoke test",
+ // Target the default code-defined agent. Drop this to use the `query` agent
+ // (the parent directory name).
+ agent: "helper",
+ async test(t) {
+ await t.send("What is 2 + 2?");
+ t.succeeded(); // gate: the turn completed
+ t.check(t.reply, includes("4")).soft(); // tracked metric, won't fail the gate
+ },
+});

apps/dev-playground/config/agents/query/evals/tool-call.eval.ts

@@ -0,0 +1,16 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * Tool-call eval: the default `helper` agent should call its `get_weather`
+ * tool when asked about the weather. (Targets `helper` explicitly — the parent
+ * directory only determines discovery, not which agent runs.)
+ */
+export default defineEval({
+ description: "Helper agent calls the get_weather tool",
+ agent: "helper",
+ async test(t) {
+ await t.send("What's the weather in Brooklyn?");
+ t.succeeded();
+ t.calledTool("get_weather");
+ },
+});