From cee5c0a746d0cf6c5af7c1f72706649a307b8d23 Mon Sep 17 00:00:00 2001 From: Wooseong Kim Date: Thu, 6 Aug 2026 18:08:10 +0900 Subject: [PATCH] fix(session): add pre-send token budget guard and compaction fallback (#269) Estimate the request token count before sending and force compaction when it would exceed the model context window, even if reported activeTokens undercounts (e.g. large tool outputs). compactSession now falls back to truncating the oldest messages without a summary if the summarization call itself fails, so the session can continue instead of deadlocking on repeated 400 errors. (cherry picked from commit ea97c568b56f0b5d7e458d727950b3e75bbc4deb) --- .../src/common/openai-message-converter.ts | 44 +++++++++++ packages/core/src/session.ts | 79 ++++++++++++------- .../tests/openai-message-converter.test.ts | 41 +++++++++- 3 files changed, 135 insertions(+), 29 deletions(-) diff --git a/packages/core/src/common/openai-message-converter.ts b/packages/core/src/common/openai-message-converter.ts index 97999045..24ea5abf 100644 --- a/packages/core/src/common/openai-message-converter.ts +++ b/packages/core/src/common/openai-message-converter.ts @@ -276,3 +276,47 @@ export class OpenAIMessageConverter { ); } } + +/** + * Estimate the token count of an OpenAI messages array (chars / 4 heuristic). + * Used as a pre-send budget guard so requests never exceed the model context + * window even when the reported usage (activeTokens) undercounts. (#269) + */ +export function estimateOpenAIMessagesTokens(messages: ChatCompletionMessageParam[]): number { + let chars = 0; + for (const message of messages) { + chars += estimateMessageChars(message); + } + return Math.max(1, Math.ceil(chars / 4)); +} + +function estimateMessageChars(message: ChatCompletionMessageParam): number { + let chars = 0; + const content = message.content; + if (typeof content === "string") { + chars += content.length; + } else if (Array.isArray(content)) { + for (const part of content) { + if (part.type === "text") { + chars += part.text.length; + } else if (part.type === "image_url") { + const url = (part as { image_url?: { url?: unknown } }).image_url?.url; + if (typeof url === "string") { + chars += url.length; + } + } + } + } + + const params = message as unknown as { tool_calls?: unknown[]; tool_call_id?: string; name?: string }; + if (params.name) { + chars += params.name.length; + } + if (params.tool_call_id) { + chars += params.tool_call_id.length; + } + if (Array.isArray(params.tool_calls)) { + chars += JSON.stringify(params.tool_calls).length; + } + return chars; +} diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index b9252eaa..4f9b6f1f 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -30,6 +30,7 @@ import { import { McpManager } from "./mcp/mcp-manager"; import { getDefaultAutoCompactWindow, + getDefaultContextWindow, type McpServerConfig, type PermissionScope, type PermissionSettings, @@ -54,7 +55,7 @@ import { } from "./common/permissions"; import { clearSessionWorkingDir } from "./tools/bash-handler"; import { reportNewPrompt } from "./common/telemetry"; -import { OpenAIMessageConverter } from "./common/openai-message-converter"; +import { estimateOpenAIMessagesTokens, OpenAIMessageConverter } from "./common/openai-message-converter"; import { supportsMultimodal } from "./common/model-capabilities"; export type { PermissionScope } from "./settings"; @@ -1395,9 +1396,18 @@ ${agentInstructions} } } + let requestMessages = this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)); + let messages = this.messageConverter.buildMessages(requestMessages, thinkingEnabled, model); const compactPromptTokenThreshold = this.getResolvedSettings().autoCompactWindow ?? getCompactPromptTokenThreshold(model); - if (session.activeTokens > compactPromptTokenThreshold) { + const estimatedTokens = estimateOpenAIMessagesTokens(messages); + const contextWindow = this.getResolvedSettings().contextWindow ?? getDefaultContextWindow(model); + // Pre-send token budget guard (#269): compact when the reported + // activeTokens exceed the auto-compact window OR when the estimated + // request size would exceed the model context window (the estimate + // catches cases where activeTokens undercounts, e.g. large tool + // outputs stored in history). + if (session.activeTokens > compactPromptTokenThreshold || estimatedTokens > contextWindow) { const message = this.buildAssistantMessage( sessionId, "The conversation is getting long, compacting...", @@ -1406,13 +1416,11 @@ ${agentInstructions} message.meta = { asThinking: true }; this.onAssistantMessage(message, false); await this.compactSession(sessionId, sessionController.signal); + // Rebuild messages after compaction so the request stays within budget. + requestMessages = this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)); + messages = this.messageConverter.buildMessages(requestMessages, thinkingEnabled, model); } - const messages = this.messageConverter.buildMessages( - this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)), - thinkingEnabled, - model - ); const thinkingOptions = buildThinkingRequestOptions(thinkingEnabled, baseURL, reasoningEffort); const response = await this.createChatCompletionStream( client, @@ -1590,30 +1598,45 @@ ${agentInstructions} const compactPrompt = getCompactPrompt(sessionMessages.slice(startIndex, endIndex)); const thinkingOptions = buildThinkingRequestOptions(thinkingEnabled, baseURL, reasoningEffort); - const response = await this.createChatCompletionStream( - client, - { - model, - ...(temperature !== undefined ? { temperature } : {}), - messages: [{ role: "user", content: compactPrompt }], - ...thinkingOptions, - }, - signal ? { signal } : undefined, - sessionId, - { - enabled: debugLogEnabled, - location: "SessionManager.compactSession", - baseURL, - params: { temperature, thinkingEnabled, reasoningEffort }, + + let compactedSummary = ""; + let responseUsage: ModelUsage | null = null; + try { + const response = await this.createChatCompletionStream( + client, + { + model, + ...(temperature !== undefined ? { temperature } : {}), + messages: [{ role: "user", content: compactPrompt }], + ...thinkingOptions, + }, + signal ? { signal } : undefined, + sessionId, + { + enabled: debugLogEnabled, + location: "SessionManager.compactSession", + baseURL, + params: { temperature, thinkingEnabled, reasoningEffort }, + } + ); + this.throwIfAborted(signal); + const rawLlmResponse = response.choices?.[0]?.message?.content; + const llmResponse = typeof rawLlmResponse === "string" ? rawLlmResponse : ""; + compactedSummary = llmResponse.replace(/[\s\S]*?<\/analysis>/gi, "").trim(); + responseUsage = response.usage ?? null; + } catch (error) { + // Degraded compaction (#269): if the summarization call itself fails + // (e.g. the request already exceeds the model context limit), still + // drop the oldest messages so the session can continue instead of + // deadlocking on repeated 400 errors. + if (signal?.aborted || this.isInterrupted(sessionId)) { + throw error; } - ); - this.throwIfAborted(signal); - const rawLlmResponse = response.choices?.[0]?.message?.content; - const llmResponse = typeof rawLlmResponse === "string" ? rawLlmResponse : ""; - const compactedSummary = llmResponse.replace(/[\s\S]*?<\/analysis>/gi, "").trim(); + compactedSummary = + "(conversation summary unavailable — earlier messages were truncated to stay within the context window.)"; + } const now = new Date().toISOString(); - const responseUsage = response.usage ?? null; this.updateSessionEntry(sessionId, (entry) => ({ ...entry, usage: accumulateUsage(entry.usage, responseUsage), diff --git a/packages/core/src/tests/openai-message-converter.test.ts b/packages/core/src/tests/openai-message-converter.test.ts index a54c213d..f7e900f4 100644 --- a/packages/core/src/tests/openai-message-converter.test.ts +++ b/packages/core/src/tests/openai-message-converter.test.ts @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { OpenAIMessageConverter } from "../common/openai-message-converter"; +import type { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { estimateOpenAIMessagesTokens, OpenAIMessageConverter } from "../common/openai-message-converter"; import type { SessionMessage } from "../session"; // --------------------------------------------------------------------------- @@ -506,3 +507,41 @@ test("OpenAIMessageConverter.findToolFunction handles null/empty toolCalls", () const toolCalls = [null, undefined, { noId: true }]; assert.equal(c.findToolFunction(toolCalls as unknown[], "call-1"), null); }); + +// --------------------------------------------------------------------------- +// estimateOpenAIMessagesTokens (#269) +// --------------------------------------------------------------------------- + +test("estimateOpenAIMessagesTokens estimates tokens from text content", () => { + const messages = [ + { role: "user", content: "x".repeat(400) }, + { role: "assistant", content: "y".repeat(200) }, + ] as unknown as ChatCompletionMessageParam[]; + assert.equal(estimateOpenAIMessagesTokens(messages), Math.ceil(600 / 4)); +}); + +test("estimateOpenAIMessagesTokens counts image_url data URIs", () => { + const messages = [ + { + role: "user", + content: [ + { type: "text", text: "hi" }, + { type: "image_url", image_url: { url: "data:image/png;base64," + "A".repeat(100) } }, + ], + }, + ] as unknown as ChatCompletionMessageParam[]; + // 2 chars text + 122 chars data-URI ("data:image/png;base64," is 22 chars) => 124 chars + assert.equal(estimateOpenAIMessagesTokens(messages), Math.ceil(124 / 4)); +}); + +test("estimateOpenAIMessagesTokens includes tool call payloads", () => { + const messages = [ + { + role: "assistant", + content: "", + tool_calls: [{ id: "call-1", type: "function", function: { name: "read", arguments: '{"file_path":"/tmp/a"}' } }], + }, + ] as unknown as ChatCompletionMessageParam[]; + const tokens = estimateOpenAIMessagesTokens(messages); + assert.ok(tokens >= 1); +});