Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 69 additions & 3 deletions apps/server/src/services/threads/title-generation.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import { renderTemplate } from "@bb/templates";
import { getThread, updateThread } from "@bb/db";
import type { PromptInput } from "@bb/domain";
import {
removeCommandMentionsFromPromptInput,
type PromptInput,
type PromptMentionCommandTrigger,
} from "@bb/domain";
import type { AppDeps, LoggedWorkSessionDeps } from "../../types.js";
import { Type } from "@earendil-works/pi-ai";
import {
Expand All @@ -10,6 +14,7 @@ import {
} from "../ai/inference.js";

const MIN_TITLE_GENERATION_WORDS = 5;
const MAX_PROMPT_TEXT_LENGTH = 80;
const MAX_GENERATED_TITLE_WORDS = 5;
const MAX_BRANCH_SLUG_LENGTH = 48;

Expand Down Expand Up @@ -55,12 +60,64 @@ function cleanPromptText(input: PromptInput[]): string {
.trim();
}

function clampPromptText(text: string): string {
return text.length <= MAX_PROMPT_TEXT_LENGTH
? text
: `${text.slice(0, MAX_PROMPT_TEXT_LENGTH - 3)}...`;
}

export function deriveTitleFallback(input: PromptInput[]): string | null {
const text = cleanPromptText(input);
if (text.length === 0) {
return null;
}
return text.length <= 80 ? text : `${text.slice(0, 77)}...`;
return clampPromptText(text);
}

interface InvokedPromptCommand {
name: string;
trigger: PromptMentionCommandTrigger;
}

export function collectInvokedPromptCommands(
input: PromptInput[],
): InvokedPromptCommand[] {
const seen = new Set<string>();
return input.flatMap((part) =>
part.type === "text"
? part.mentions.flatMap((mention) => {
if (mention.resource.kind !== "command") {
return [];
}
const { name, trigger } = mention.resource;
const key = `${trigger}${name}`;
if (seen.has(key)) {
return [];
}
seen.add(key);
return [{ name, trigger }];
})
: [],
);
}

function promptTextWithoutCommands(
input: PromptInput[],
commands: InvokedPromptCommand[],
): string {
return cleanPromptText(
commands.reduce<PromptInput[]>(
(remaining, command) =>
removeCommandMentionsFromPromptInput(remaining, command),
input,
),
);
}

function formatInvokedCommands(commands: InvokedPromptCommand[]): string {
return commands
.map((command) => `${command.trigger}${command.name}`)
.join(", ");
}

export function shouldGenerateThreadTitle(input: PromptInput[]): boolean {
Expand All @@ -69,6 +126,10 @@ export function shouldGenerateThreadTitle(input: PromptInput[]): boolean {
return false;
}

if (collectInvokedPromptCommands(input).length > 0) {
return true;
}

return text.split(/\s+/u).length >= MIN_TITLE_GENERATION_WORDS;
}

Expand Down Expand Up @@ -137,8 +198,13 @@ export async function generateThreadMetadataWithOutcome(
return complete(null, "too-short");
}

const commands = collectInvokedPromptCommands(args.input);
const body = promptTextWithoutCommands(args.input, commands);
const prompt = renderTemplate("generateThreadMetadata", {
cleanedPrompt: fallback,
cleanedPrompt: body.length > 0 ? clampPromptText(body) : fallback,
...(commands.length > 0
? { invokedCommands: formatInvokedCommands(commands) }
: {}),
});
const maxAttempts = Math.max(1, args.timeoutMaxAttempts ?? 1);

Expand Down
25 changes: 25 additions & 0 deletions apps/server/test/helpers/prompt-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,28 @@ function textPrompt(text: string): PromptInput {
export function textInput(text: string): PromptInput[] {
return [textPrompt(text)];
}

export function skillInput(name: string, rest = ""): PromptInput[] {
const command = `/${name}`;
return [
{
type: "text",
text: `${command}${rest}`,
mentions: [
{
start: 0,
end: command.length,
resource: {
kind: "command",
trigger: "/",
name,
source: "skill",
origin: "user",
label: name,
argumentHint: null,
},
},
],
},
];
}
44 changes: 43 additions & 1 deletion apps/server/test/threads/generated-thread-titles.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
waitForQueuedCommandAfter,
} from "../helpers/commands.js";
import { readJson } from "../helpers/json.js";
import { textInput } from "../helpers/prompt-input.js";
import { skillInput, textInput } from "../helpers/prompt-input.js";
import {
seedEnvironment,
seedHostSession,
Expand Down Expand Up @@ -924,6 +924,45 @@ describe("generated thread titles", () => {
});
});

it("titles a skill invocation from the task, not the command token", async () => {
mockThreadMetadata({ title: "Drop stale release branches" });
await withTestHarness(async (harness) => {
await expect(
generateThreadMetadataWithOutcome(harness.deps, {
input: skillInput("sync-repo", " and drop the stale release branches"),
threadId: "thr_skill_metadata",
}),
).resolves.toMatchObject({
metadata: { title: "Drop stale release branches" },
});
const prompt = piAiMocks.complete.mock.calls[0]?.[1].messages[0].content;
expect(prompt).toContain("and drop the stale release branches");
expect(prompt).toContain(
"The prompt invokes these commands or skills: /sync-repo.",
);
expect(prompt).not.toContain("/sync-repo and drop");
});
});

it("titles a bare skill invocation from what the skill does", async () => {
mockThreadMetadata({ title: "Generate the weekly report" });
await withTestHarness(async (harness) => {
await expect(
generateThreadMetadataWithOutcome(harness.deps, {
input: skillInput("weekly-report"),
threadId: "thr_bare_skill_metadata",
}),
).resolves.toMatchObject({
metadata: { title: "Generate the weekly report" },
});
expect(
piAiMocks.complete.mock.calls[0]?.[1].messages[0].content,
).toContain(
"The prompt invokes these commands or skills: /weekly-report.",
);
});
});

it("does not retry non-transient metadata inference failures", async () => {
piAiMocks.getModel.mockReturnValue({ provider: "test" });
piAiMocks.complete.mockRejectedValue(new Error("metadata failed"));
Expand All @@ -940,6 +979,9 @@ describe("generated thread titles", () => {
reason: "failed",
});
expect(piAiMocks.complete).toHaveBeenCalledTimes(1);
expect(
piAiMocks.complete.mock.calls[0]?.[1].messages[0].content,
).not.toContain("The prompt invokes these commands or skills");
});
});
});
51 changes: 51 additions & 0 deletions apps/server/test/threads/title-generation.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import type { PromptInput } from "@bb/domain";
import {
collectInvokedPromptCommands,
deriveTitleFallback,
sanitizeGeneratedTitle,
shouldGenerateThreadTitle,
Expand All @@ -14,6 +15,29 @@ function textInput(text: string): PromptInput {
};
}

function skillInput(name: string, rest = ""): PromptInput {
const command = `/${name}`;
return {
type: "text",
text: `${command}${rest}`,
mentions: [
{
start: 0,
end: command.length,
resource: {
kind: "command",
trigger: "/",
name,
source: "skill",
origin: "user",
label: name,
argumentHint: null,
},
},
],
};
}

describe("thread title generation", () => {
it("does not generate titles for inputs shorter than five words", () => {
expect(shouldGenerateThreadTitle([textInput("fix")])).toBe(false);
Expand Down Expand Up @@ -55,6 +79,33 @@ describe("thread title generation", () => {
expect(sanitizeGeneratedTitle(" ")).toBeNull();
});

it("generates titles for invoked skills regardless of prompt length", () => {
expect(shouldGenerateThreadTitle([skillInput("sync-repo")])).toBe(true);
expect(
shouldGenerateThreadTitle([skillInput("sync-repo", " then deploy")]),
).toBe(true);
});

it("keeps the raw command text as the fallback for invoked skills", () => {
expect(deriveTitleFallback([skillInput("sync-repo", " then deploy")])).toBe(
"/sync-repo then deploy",
);
});

it("collects each invoked command once, in prompt order", () => {
expect(
collectInvokedPromptCommands([
skillInput("sync-repo"),
textInput("then"),
skillInput("review-diff"),
skillInput("sync-repo"),
]),
).toEqual([
{ name: "sync-repo", trigger: "/" },
{ name: "review-diff", trigger: "/" },
]);
});

it("keeps fallback derivation independent from title generation eligibility", () => {
const input = [textInput("fix bug")];

Expand Down
5 changes: 5 additions & 0 deletions packages/templates/src/templates/generate-thread-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,17 @@ intent: Generate stable, operator-friendly metadata for threads without adding e
editingNotes: Callers use tool-call structured output; the model calls a `result` tool with the schema.
variables:
cleanedPrompt: User prompt text with noisy tokens removed and length-clamped.
invokedCommands?: Comma-separated slash commands or skills the prompt invokes, when it invokes any.
---
You create concise titles for coding tasks.
Call the `result` tool with:
- title: short, clear, 4-5 words maximum, sentence case

Consider the user's intent when titling to make it useful. For instance, if they detail specific tools to use to solve a problem, it is the problem that should be the title, not the tools that should be used.

{{#if invokedCommands}}
The prompt invokes these commands or skills: {{invokedCommands}}. They name how the work is carried out, so title the work they are applied to. When the prompt names nothing else, title what the invoked command itself does.

{{/if}}
Task:
{{cleanedPrompt}}
Loading