diff --git a/apps/core/src/agent/loop.ts b/apps/core/src/agent/loop.ts index 79c88158..6e7b1f1a 100644 --- a/apps/core/src/agent/loop.ts +++ b/apps/core/src/agent/loop.ts @@ -43,6 +43,9 @@ import { invalidateProjectContext, } from "../context/tree-cache.js"; import { MemoryService, renderPromptMemoryContext } from "../compaction/index.js"; +import { createLlmSummarizer } from "../compaction/llm-summarizer.js"; +import type { CompactOptions } from "../compaction/service.js"; +import { getModelContextLimit } from "../models-dev.js"; import { getProvider } from "../providers/index.js"; import { createHookRuntime, type HookRuntime } from "../hooks/runtime.js"; import type { HookResult } from "../agent/types.js"; @@ -392,6 +395,40 @@ export class AgentLoop { } } + // Resolve the model's real context window from models.dev so compaction + // fires against the actual limit (200K/1M) rather than a conservative + // fallback. Returns undefined on lookup failure (offline / unknown model), + // which makes shouldCompact fall back to the local model table. + private async resolveContextLimit( + provider: string, + model: string | undefined, + ): Promise { + if (!model) return undefined; + try { + const limit = await getModelContextLimit(provider, model); + return limit > 0 ? limit : undefined; + } catch { + return undefined; + } + } + + // Build compaction options that summarize via the active provider/model. + // Cancellable through the loop's AbortController. On provider-lookup failure + // returns {}, so MemoryService.compact falls back to the heuristic summary. + private compactOptions( + provider: string, + model: string | undefined, + ): CompactOptions { + try { + const aiProvider = getProvider(provider as any); + return { + llmSummarize: createLlmSummarizer(aiProvider, model, this.abort.signal), + }; + } catch { + return {}; + } + } + // =========================================================================== // PRIVATE: executeTurn() // One iteration: build prompt → send to provider → normalize → parse → execute @@ -511,9 +548,12 @@ export class AgentLoop { if (toolCalls.length === 0) { this.memory.addMessage("assistant", providerResult.content); await this.appendAssistantMessage(providerResult.content); - if (this.memory.shouldCompact(model ?? provider)) { + const contextLimit = await this.resolveContextLimit(provider, model); + if (this.memory.shouldCompact(model ?? provider, contextLimit)) { // PreCompact/PostCompact hooks run inside MemoryService.compact() - const result = await this.memory.compact(); + const result = await this.memory.compact( + this.compactOptions(provider, model), + ); if (result.success && result.summary) { this.recorder.recordCompactOccurred( result.tokenCountBefore, @@ -576,9 +616,12 @@ export class AgentLoop { providerResult.content || `[Executed ${toolCalls.length} tools]`, ); - if (this.memory.shouldCompact(model ?? provider)) { + const contextLimit = await this.resolveContextLimit(provider, model); + if (this.memory.shouldCompact(model ?? provider, contextLimit)) { // PreCompact/PostCompact hooks run inside MemoryService.compact() - const result = await this.memory.compact(); + const result = await this.memory.compact( + this.compactOptions(provider, model), + ); if (result.success && result.summary) { this.recorder.recordCompactOccurred( result.tokenCountBefore, diff --git a/apps/core/src/compaction/index.ts b/apps/core/src/compaction/index.ts index c0c8ec12..5983a5aa 100644 --- a/apps/core/src/compaction/index.ts +++ b/apps/core/src/compaction/index.ts @@ -2,5 +2,6 @@ export * from "./types.js"; export * from "./tokens.js"; export * from "./selector.js"; export * from "./summarizer.js"; +export * from "./llm-summarizer.js"; export * from "./storage.js"; export * from "./service.js"; diff --git a/apps/core/src/compaction/llm-summarizer.ts b/apps/core/src/compaction/llm-summarizer.ts new file mode 100644 index 00000000..fb3bdf94 --- /dev/null +++ b/apps/core/src/compaction/llm-summarizer.ts @@ -0,0 +1,61 @@ +import type { AIProvider } from "../providers/types.js"; +import type { MemoryMessage } from "./types.js"; +import type { SummarizeInput } from "./summarizer.js"; + +/** + * Produces the markdown body of a compaction summary. Throws on failure so the + * caller can fall back to the heuristic summarizer. + */ +export type LlmSummarize = (input: SummarizeInput) => Promise; + +const SYSTEM_PROMPT = `You compress a coding-assistant session into a concise handoff summary so work can continue after older messages are dropped. + +Output GitHub-flavored markdown using exactly these section headers, in this order: +## Goal +## Done +## In Progress +## Blocked +## Decisions +## Relevant Files +## Next Steps + +Rules: +- Be factual and terse. Do not invent progress that isn't in the transcript. +- Preserve exact file paths, identifiers, commands, and unresolved errors. +- Under "Blocked", list only real, still-open blockers. +- If a section has nothing, write "- (none)".`; + +function renderTranscript(messages: MemoryMessage[]): string { + return messages.map((m) => `${m.role}: ${m.content}`).join("\n\n"); +} + +export function createLlmSummarizer( + provider: AIProvider, + model: string | undefined, + abortSignal?: AbortSignal, +): LlmSummarize { + return async (input) => { + const parts: string[] = []; + if (input.previousSummary) { + parts.push( + `Previous summary (carry forward and update, don't repeat verbatim):\n\n${input.previousSummary}`, + ); + } + parts.push( + `Session transcript to summarize:\n\n${renderTranscript(input.messages)}`, + ); + + const result = await provider.execute({ + system: SYSTEM_PROMPT, + prompt: parts.join("\n\n---\n\n"), + model, + temperature: 0, + maxTokens: 1_024, + abortSignal, + }); + + const content = result.content?.trim(); + if (!content) throw new Error("provider returned an empty summary"); + return content; + }; +} diff --git a/apps/core/src/compaction/service.ts b/apps/core/src/compaction/service.ts index c9546811..8a859304 100644 --- a/apps/core/src/compaction/service.ts +++ b/apps/core/src/compaction/service.ts @@ -11,9 +11,17 @@ import { } from "./types.js"; import { estimateTokenCount, shouldCompact } from "./tokens.js"; import { selectForCompaction } from "./selector.js"; -import { summarizeMessages } from "./summarizer.js"; +import { makeSummary, summarizeMessages } from "./summarizer.js"; +import type { LlmSummarize } from "./llm-summarizer.js"; import { FileMemoryStorage, type MemoryStorage } from "./storage.js"; +export interface CompactOptions { + // When provided, used to generate the summary; on failure the heuristic + // summarizer is used instead. The agent loop supplies this with the current + // provider/model so summaries are real LLM output rather than keyword bullets. + llmSummarize?: LlmSummarize; +} + interface MemoryServiceOptions { config?: Partial; storage?: MemoryStorage; @@ -68,7 +76,9 @@ export class MemoryService { return message; } - shouldCompact(model: string): boolean { + // `contextLimit` comes from models.dev (getModelContextLimit) when available; + // omit it to fall back to the local model table in tokens.ts. + shouldCompact(model: string, contextLimit?: number): boolean { if ( this.blockedAtTokenCount !== undefined && this.state.tokenCount < @@ -80,6 +90,7 @@ export class MemoryService { this.state.tokenCount, model, this.config.autoCompactBufferTokens, + contextLimit, ); } @@ -91,7 +102,7 @@ export class MemoryService { }; } - async compact(): Promise { + async compact(options: CompactOptions = {}): Promise { const selected = selectForCompaction(this.state.messages, this.config); if (selected.summarize.length === 0) { return { @@ -121,11 +132,23 @@ export class MemoryService { } const previousSummary = this.state.summaries.at(-1)?.content; - const summary = summarizeMessages({ + const summarizeInput = { sessionId: this.state.sessionId, previousSummary, messages: selected.summarize, - }); + }; + let summary = summarizeMessages(summarizeInput); + if (options.llmSummarize) { + try { + const content = await options.llmSummarize(summarizeInput); + summary = makeSummary(summarizeInput, content); + } catch (err) { + // Fall back to the heuristic summary already computed above. + console.warn( + `[MemoryService] LLM summarization failed, using heuristic: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } const tokenCountAfter = summary.summaryTokenCount + selected.preserveTokenCount; diff --git a/apps/core/src/compaction/storage.ts b/apps/core/src/compaction/storage.ts index f4cf41cc..b3c92c56 100644 --- a/apps/core/src/compaction/storage.ts +++ b/apps/core/src/compaction/storage.ts @@ -3,14 +3,20 @@ import { mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, statSync, writeFileSync, } from "node:fs"; +import { homedir } from "node:os"; import { join } from "node:path"; import type { MemoryState } from "./types.js"; -const SESSION_DIR = ".freecode/sessions"; +// Home-rooted like every other ~/.freecode subsystem (sessions, rollout, +// state, config). Previously this defaulted to process.cwd(), which scattered +// compaction memory into whatever directory core was launched from (e.g. the +// repo root) and orphaned it from the session it belongs to. +const MEMORY_ROOT = join(homedir(), ".freecode", "memory"); export interface MemoryStorage { save(state: MemoryState): void; @@ -20,10 +26,10 @@ export interface MemoryStorage { } export class FileMemoryStorage implements MemoryStorage { - constructor(private readonly basePath = process.cwd()) {} + constructor(private readonly basePath = MEMORY_ROOT) {} private sessionDir(sessionId: string): string { - return join(this.basePath, SESSION_DIR, sessionId); + return join(this.basePath, sessionId); } private memoryPath(sessionId: string): string { @@ -32,10 +38,13 @@ export class FileMemoryStorage implements MemoryStorage { save(state: MemoryState): void { mkdirSync(this.sessionDir(state.sessionId), { recursive: true }); - writeFileSync( - this.memoryPath(state.sessionId), - JSON.stringify(state, null, 2), - ); + // Atomic write: a crash mid-write must not corrupt memory.json (load() + // treats a corrupt file as "no memory" and discards the whole session's + // state). Write to a temp file, then rename — rename is atomic on POSIX. + const dest = this.memoryPath(state.sessionId); + const tmp = `${dest}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(state, null, 2)); + renameSync(tmp, dest); } load(sessionId: string): MemoryState | undefined { @@ -50,10 +59,9 @@ export class FileMemoryStorage implements MemoryStorage { } listSessions(): string[] { - const dir = join(this.basePath, SESSION_DIR); - if (!existsSync(dir)) return []; - return readdirSync(dir).filter((entry) => - statSync(join(dir, entry)).isDirectory(), + if (!existsSync(this.basePath)) return []; + return readdirSync(this.basePath).filter((entry) => + statSync(join(this.basePath, entry)).isDirectory(), ); } diff --git a/apps/core/src/compaction/summarizer.ts b/apps/core/src/compaction/summarizer.ts index 5b2dfd52..64549f9a 100644 --- a/apps/core/src/compaction/summarizer.ts +++ b/apps/core/src/compaction/summarizer.ts @@ -1,12 +1,34 @@ import type { CompactionSummary, MemoryMessage } from "./types.js"; import { estimateTokenCount } from "./tokens.js"; -interface SummarizeInput { +export interface SummarizeInput { sessionId: string; previousSummary?: string; messages: MemoryMessage[]; } +/** + * Wrap summary content (from either the heuristic or an LLM) into a + * CompactionSummary, computing the token bookkeeping consistently. + */ +export function makeSummary( + input: SummarizeInput, + content: string, +): CompactionSummary { + const originalTokenCount = input.messages.reduce( + (sum, message) => sum + message.tokenCount, + 0, + ); + return { + id: `summary-${input.sessionId}-${Date.now()}`, + createdAt: Date.now(), + originalMessageCount: input.messages.length, + originalTokenCount, + summaryTokenCount: estimateTokenCount(content), + content, + }; +} + function clip(text: string, maxChars: number): string { return text.length <= maxChars ? text : `${text.slice(0, maxChars)}...`; } @@ -47,11 +69,15 @@ function extractWorkStatus(messages: MemoryMessage[]): { const content = msg.content.toLowerCase(); const summary = clip(msg.content, 120); - if ( - content.includes("blocked") || - content.includes("waiting") || - content.includes("error") - ) { + // Word-boundary match so "no errors"/"error handling" don't count as a + // blocker, and negations ("no error", "without errors") are excluded. + const isBlocked = + /\b(blocked|waiting|errors?|failed|failing)\b/.test(content) && + !/\b(no|without|zero|resolved|fixed)\s+(errors?|blockers?|issues?)\b/.test( + content, + ); + + if (isBlocked) { if (!blocked.includes(summary)) blocked.push(summary); } else if ( content.includes("working on") || @@ -187,18 +213,5 @@ export function summarizeMessages(input: SummarizeInput): CompactionSummary { lines.push("- Continue from where the session left off"); } - const content = lines.join("\n"); - const originalTokenCount = input.messages.reduce( - (sum, message) => sum + message.tokenCount, - 0, - ); - - return { - id: `summary-${input.sessionId}-${Date.now()}`, - createdAt: Date.now(), - originalMessageCount: input.messages.length, - originalTokenCount, - summaryTokenCount: estimateTokenCount(content), - content, - }; + return makeSummary(input, lines.join("\n")); } diff --git a/apps/core/src/compaction/tokens.test.ts b/apps/core/src/compaction/tokens.test.ts index fffdeac9..392edce5 100644 --- a/apps/core/src/compaction/tokens.test.ts +++ b/apps/core/src/compaction/tokens.test.ts @@ -11,15 +11,22 @@ test("estimateTokenCount uses a conservative char estimate", () => { assert.equal(estimateTokenCount("Hello World"), 3); }); -test("getContextLimit falls back for unknown models", () => { +test("getContextLimit returns the offline fallback floor", () => { assert.equal(getContextLimit("unknown-model"), 100_000); + assert.equal(getContextLimit("gpt-4o"), 100_000); }); test("getAutoCompactThreshold reserves compaction buffer", () => { - assert.equal(getAutoCompactThreshold("gpt-4o", 13_000), 115_000); + assert.equal(getAutoCompactThreshold("gpt-4o", 13_000), 87_000); }); -test("shouldCompact is only true at or above threshold", () => { - assert.equal(shouldCompact(114_999, "gpt-4o", 13_000), false); - assert.equal(shouldCompact(115_000, "gpt-4o", 13_000), true); +test("shouldCompact uses the fallback limit when no explicit limit is given", () => { + assert.equal(shouldCompact(86_999, "gpt-4o", 13_000), false); + assert.equal(shouldCompact(87_000, "gpt-4o", 13_000), true); +}); + +test("shouldCompact prefers an explicit (models.dev) context limit", () => { + // 200K model with a 13K buffer → threshold 187K, not the 87K fallback. + assert.equal(shouldCompact(150_000, "gpt-4o", 13_000, 200_000), false); + assert.equal(shouldCompact(187_000, "gpt-4o", 13_000, 200_000), true); }); diff --git a/apps/core/src/compaction/tokens.ts b/apps/core/src/compaction/tokens.ts index 7c646bdb..46efe1e8 100644 --- a/apps/core/src/compaction/tokens.ts +++ b/apps/core/src/compaction/tokens.ts @@ -1,32 +1,22 @@ const CHARS_PER_TOKEN = 4; -export const MODEL_CONTEXT_LIMITS: Record = { - "gpt-4o": 128_000, - "gpt-4-turbo": 128_000, - "gpt-4": 8_192, - "gpt-3.5-turbo": 16_385, - "claude-3-5-sonnet": 200_000, - "claude-3-opus": 200_000, - "claude-3-sonnet": 200_000, - "gemini-1.5-pro": 1_000_000, - "gemini-1.5-flash": 1_000_000, - default: 100_000, -}; +// Conservative offline floor. The live source of truth is models.dev, resolved +// via getModelContextLimit() and passed into shouldCompact() as an explicit +// limit; this constant is used only when that lookup returns nothing (no +// network, cold cache, or unknown model). A per-model table here would just +// drift out of date — being wrong only makes us compact slightly early, never +// lose data, so a single safe value is enough. +export const FALLBACK_CONTEXT_LIMIT = 100_000; export function estimateTokenCount(text: string): number { if (text.length === 0) return 0; return Math.ceil(text.length / CHARS_PER_TOKEN); } -export function getContextLimit(model: string): number { - const exact = MODEL_CONTEXT_LIMITS[model]; - if (exact !== undefined) return exact; - // Prefix match so versioned ids ("claude-3-5-sonnet-20241022") resolve. - // Longest key first so "gpt-4o" wins over "gpt-4". - const prefix = Object.keys(MODEL_CONTEXT_LIMITS) - .filter((key) => model.startsWith(key)) - .sort((a, b) => b.length - a.length)[0]; - return prefix ? MODEL_CONTEXT_LIMITS[prefix] : MODEL_CONTEXT_LIMITS.default; +// Kept for callers/tests; always returns the offline floor now that the live +// limit comes from models.dev via shouldCompact's contextLimit argument. +export function getContextLimit(_model: string): number { + return FALLBACK_CONTEXT_LIMIT; } export function getAutoCompactThreshold( @@ -40,6 +30,10 @@ export function shouldCompact( tokenCount: number, model: string, bufferTokens: number, + contextLimit?: number, ): boolean { - return tokenCount >= getAutoCompactThreshold(model, bufferTokens); + // Prefer an explicit (models.dev) limit; fall back to the local table. + const limit = + contextLimit && contextLimit > 0 ? contextLimit : getContextLimit(model); + return tokenCount >= Math.max(0, limit - bufferTokens); } diff --git a/apps/core/src/compaction/types.ts b/apps/core/src/compaction/types.ts index 6e90c51c..817db457 100644 --- a/apps/core/src/compaction/types.ts +++ b/apps/core/src/compaction/types.ts @@ -28,9 +28,7 @@ export interface MemoryState { export interface CompactionConfig { autoCompactBufferTokens: number; - warningBufferTokens: number; preserveRecentTurns: number; - minPreserveRecentTokens: number; maxPreserveRecentTokens: number; maxToolOutputChars: number; } @@ -61,9 +59,7 @@ export interface PromptMemoryContext { export const DEFAULT_COMPACTION_CONFIG: CompactionConfig = { autoCompactBufferTokens: 13_000, - warningBufferTokens: 20_000, preserveRecentTurns: 2, - minPreserveRecentTokens: 2_000, maxPreserveRecentTokens: 8_000, maxToolOutputChars: 2_000, }; diff --git a/apps/web/app/internal/components/presentation/node-content/MemoryNodeContent.tsx b/apps/web/app/internal/components/presentation/node-content/MemoryNodeContent.tsx index c3083f0a..ea0cd2a3 100644 --- a/apps/web/app/internal/components/presentation/node-content/MemoryNodeContent.tsx +++ b/apps/web/app/internal/components/presentation/node-content/MemoryNodeContent.tsx @@ -8,39 +8,39 @@ import { NodeHeader } from "./NodeHeader"; const MEMORY_FLOW_STEPS = [ { number: 1, - title: "addMessage(role, content)", + title: "1. Write down each message", description: - "MemoryService records each turn, truncates long tool output, and estimates its token cost", + "Every user prompt and model reply is saved, and we estimate its size in tokens (≈ 4 characters = 1 token). Long tool output is trimmed first so it can't hog space.", }, { number: 2, - title: "shouldCompact(model)", + title: "2. Are we running out of room?", description: - "Trigger when tokenCount passes the model's context limit minus the auto-compact buffer", + "After each turn we compare the running token count to the model's real context limit (looked up from models.dev) minus a 13,000-token safety buffer. Cross that line and compaction kicks in.", }, { number: 3, - title: "selectForCompaction()", + title: "3. Decide what to keep vs. fold away", description: - "Keep the last 2 user turns (bounded by token caps); mark everything older to summarize", + "The last 2 user turns (capped at 8,000 tokens) are kept word-for-word so recent detail survives. Everything older is marked to be summarized.", }, { number: 4, - title: "runPreCompact() hook", + title: "4. Ask permission (PreCompact hook)", description: - "PreCompact may block — if it does, retry is deferred until +5k more tokens accumulate", + "A PreCompact hook can veto the compaction. If it blocks, we back off and don't retry until another 5,000 tokens pile up — so we don't re-ask every message.", }, { number: 5, - title: "summarizeMessages()", + title: "5. Summarize the old messages", description: - "Fold old turns into an anchored summary: Goal · Done · In Progress · Files · Next Steps", + "The active model writes a short, structured recap (Goal · Done · In Progress · Blocked · Decisions · Files · Next Steps). If that call fails, a keyword-based summary is used as a fallback.", }, { number: 6, - title: "commit + runPostCompact() hook", + title: "6. Swap it in and save", description: - "Replace history with [summary + preserved turns], persist, then notify PostCompact", + "History becomes [summary + preserved recent turns], the token count drops back down, state is written to disk atomically, and a PostCompact hook is notified.", }, ]; @@ -48,21 +48,21 @@ const memoryComponents = [ { name: "MemoryService", file: "apps/core/src/compaction/service.ts", - description: "Orchestrates recording, threshold checks, and compaction", + description: "Orchestrates recording, the threshold check, and compaction", icon: Brain, color: "#f59e0b", }, { name: "FileMemoryStorage", file: "apps/core/src/compaction/storage.ts", - description: "JSON persistence of session memory state", + description: "Atomic JSON persistence at ~/.freecode/memory/{sessionId}", icon: Database, color: "#f59e0b", }, { name: "tokens.ts", file: "apps/core/src/compaction/tokens.ts", - description: "Token estimation + per-model context limits", + description: "Token estimate + the 'should we compact?' math", icon: Zap, color: "#f59e0b", }, @@ -76,7 +76,7 @@ const memoryComponents = [ { name: "summarizer.ts", file: "apps/core/src/compaction/summarizer.ts", - description: "Anchored summary carrying forward the previous one", + description: "LLM summary (with a heuristic fallback), carried forward", icon: FileText, color: "#f59e0b", }, @@ -85,9 +85,7 @@ const memoryComponents = [ const compactionStats = [ { label: "preserveRecentTurns", value: "2 turns" }, { label: "autoCompactBufferTokens", value: "13,000" }, - { label: "warningBufferTokens", value: "20,000" }, { label: "maxPreserveRecentTokens", value: "8,000" }, - { label: "minPreserveRecentTokens", value: "2,000" }, { label: "maxToolOutputChars", value: "2,000" }, ]; @@ -99,15 +97,34 @@ export function MemoryNodeContent() { subtext="Session Context Compaction" />

- The compaction system keeps long sessions inside the - model's context window. After each turn the loop asks{" "} - MemoryService whether the running token count has - crossed the model's budget; when it has, older turns are folded - into an anchored summary while the most recent turns - are preserved verbatim. The whole thing is model-aware{" "} - (context limits per model) and hook-gated (PreCompact - can veto, PostCompact is notified). A separate idle-gap micro-compaction - in the loop also trims stale tool output on cold restarts. + The problem: a model can only read so much at once — its{" "} + context window. Think of it like a whiteboard with + limited space. As a conversation grows, the whiteboard fills up, and + eventually there's no room left to write the next reply. +

+

+ Compaction is how FreeCode keeps long sessions from + overflowing. When the conversation gets close to full, it{" "} + erases the old notes and replaces them with a short summary{" "} + — while keeping the last couple of turns exactly as they were. The + session keeps going, the gist is preserved, and there's room to + write again. +

+ + {/* Worked example — the trigger is BEFORE the window is full */} +
+ EXAMPLE · 1,000,000-token window +
+ compaction fires at 1,000,000 − 13,000 = 987,000 tokens +
+ (it triggers just BEFORE full, leaving ~13k headroom for the reply) +
+ +

+ Notice it does not wait until the window is completely + full — it fires a little early, on purpose, so there's always + headroom for the model to answer. Here's the whole cycle, step by + step:

{/* Memory Flow */} @@ -144,7 +161,7 @@ export function MemoryNodeContent() { {/* Compaction Stats */}
-
Compaction Config
+
Compaction Config (the knobs)
{compactionStats.map((stat) => (