From efe9b3b8ed3185b65450351d0615f5d0b7c19608 Mon Sep 17 00:00:00 2001 From: Tim Erwin Date: Thu, 3 Sep 2026 11:47:03 +0700 Subject: [PATCH 1/5] gemini: round-trip thoughtSignature through tool replay Gemini issues an opaque thoughtSignature with every functionCall and REQUIRES it echoed back when that call is replayed in a later turn. The adapter decoded functionCall into a tool_use block but dropped the signature, so any multi-turn tool conversation failed with a 400 on the second turn. This is why code review broke: the reviewer is a tool-using agent, so it hit the 400 on its first replay, every time. It reproduces identically on Gemini 3.6, so it is an adapter bug, not a 3.8 regression. Adds toolUseFromPart as a testable decode seam, echoes the signature back in blockToPart, and pins both directions with a multi-turn replay test plus a no-signature case (an older transcript must still send none, not an empty one). --- internal/providers/gemini/gemini.go | 52 ++++++++++++++--- internal/providers/gemini/gemini_test.go | 74 ++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/internal/providers/gemini/gemini.go b/internal/providers/gemini/gemini.go index 9eaa9dd..27f4482 100644 --- a/internal/providers/gemini/gemini.go +++ b/internal/providers/gemini/gemini.go @@ -241,6 +241,36 @@ func inlineBlobPart(b wire.Block) *genai.Part { // blockToPart maps one wire.Block onto a genai.Part. Text → Text part; image // → InlineData (base64-decoded back to bytes); tool_use → FunctionCall; // tool_result → FunctionResponse; thinking → dropped (per-call only on Gemini). +// toolUseFromPart converts a Gemini functionCall part into a tool_use block. +// It is the exact inverse of blockToPart's tool_use case, and exists as its own +// function so the round-trip can be tested without a live stream — the bug it +// guards against is only reachable across TWO turns. +// +// The thought signature is the load-bearing part. Gemini 3.x issues one with +// every functionCall and REQUIRES it echoed back when that call is replayed: +// +// 400 INVALID_ARGUMENT: Function call is missing a thought_signature in +// functionCall parts. +// +// Dropping it made every tool-using Gemini turn fail on turn 2, across the +// whole model family. It is opaque provider data — stored base64 so it survives +// the JSON wire, echoed verbatim, never interpreted. +func toolUseFromPart(part *genai.Part) (wire.Block, bool) { + fc := part.FunctionCall + if fc == nil || fc.Name == "" { + return wire.Block{}, false + } + args, _ := json.Marshal(fc.Args) + if len(args) == 0 || string(args) == "null" { + args = json.RawMessage("{}") + } + blk := wire.Block{Type: "tool_use", ID: fc.ID, Name: fc.Name, Input: args} + if len(part.ThoughtSignature) > 0 { + blk.Signature = base64.StdEncoding.EncodeToString(part.ThoughtSignature) + } + return blk, true +} + func (g *Gemini) blockToPart(b wire.Block, callName map[string]string) *genai.Part { switch b.Type { case "text": @@ -259,11 +289,21 @@ func (g *Gemini) blockToPart(b wire.Block, callName map[string]string) *genai.Pa provcore.LogToolInputMalformed("gemini", err) } } - return &genai.Part{FunctionCall: &genai.FunctionCall{ + part := &genai.Part{FunctionCall: &genai.FunctionCall{ ID: b.ID, Name: b.Name, Args: args, }} + // Echo the thought signature Gemini issued for this call. Required on + // replay (see the capture site) — a missing one is a hard 400, not a + // degraded response. Absent for a call that came from another vendor, + // which is correct: there is nothing of Gemini's to return. + if b.Signature != "" { + if sig, err := base64.StdEncoding.DecodeString(b.Signature); err == nil { + part.ThoughtSignature = sig + } + } + return part case "tool_result": var resp map[string]any if b.IsError { @@ -513,14 +553,8 @@ func (g *Gemini) streamOnce(ctx context.Context, cl *genai.Client, model string, emitted = true } } - if fc := part.FunctionCall; fc != nil && fc.Name != "" { - args, _ := json.Marshal(fc.Args) - if len(args) == 0 || string(args) == "null" { - args = json.RawMessage("{}") - } - fnCalls = append(fnCalls, wire.Block{ - Type: "tool_use", ID: fc.ID, Name: fc.Name, Input: args, - }) + if blk, ok := toolUseFromPart(part); ok { + fnCalls = append(fnCalls, blk) } } } diff --git a/internal/providers/gemini/gemini_test.go b/internal/providers/gemini/gemini_test.go index a28dc12..4bef84a 100644 --- a/internal/providers/gemini/gemini_test.go +++ b/internal/providers/gemini/gemini_test.go @@ -1,6 +1,7 @@ package gemini import ( + "encoding/base64" "fmt" "testing" @@ -229,3 +230,76 @@ func TestGeminiOverflowClassification(t *testing.T) { type errString string func (e errString) Error() string { return string(e) } + +// The multi-turn tool replay that single-turn tests could never catch. +// +// Gemini 3.x issues a thoughtSignature alongside every functionCall and +// REQUIRES it echoed back when that call is replayed on the next turn. Without +// it the continuation is rejected outright: +// +// 400 INVALID_ARGUMENT: Function call is missing a thought_signature in +// functionCall parts. +// +// That made every tool-using Gemini turn die on turn 2 — the whole model +// family, not one version. It went unnoticed because the capability checks +// that ran against Gemini were all SINGLE-turn (text, vision, PDF, thinking), +// and a first turn has no prior call to replay. Verified live on Vertex against +// gemini-3.8-flash and gemini-3.6-flash: identical failure without the +// signature, success with it. +func TestGeminiRoundTripsThoughtSignature(t *testing.T) { + const sig = "AY89a1+xXXW4pP89" + encoded := base64.StdEncoding.EncodeToString([]byte(sig)) + + // DECODE: a functionCall part's signature is captured onto the block. + g := NewGemini("key") + decoded, ok := toolUseFromPart(&genai.Part{ + FunctionCall: &genai.FunctionCall{ID: "tu_1", Name: "bash", Args: map[string]any{"cmd": "ls"}}, + ThoughtSignature: []byte(sig), + }) + if !ok || decoded.Type != "tool_use" { + t.Fatalf("decode produced %+v (ok=%v), want a tool_use block", decoded, ok) + } + blocks := []wire.Block{decoded} + if blocks[0].Signature != encoded { + t.Fatalf("decoded signature = %q, want the base64 of what Gemini issued", blocks[0].Signature) + } + + // ENCODE: replaying that block sends the signature back, byte-identical. + contents := g.buildContents(wire.Request{Messages: []wire.Message{ + {Role: "user", Blocks: []wire.Block{{Type: "text", Text: "run ls"}}}, + {Role: "assistant", Blocks: []wire.Block{blocks[0]}}, + {Role: "user", Blocks: []wire.Block{{Type: "tool_result", ToolUseID: "tu_1", Name: "bash", Content: "a.txt"}}}, + }}) + if len(contents) != 3 { + t.Fatalf("buildContents produced %d contents, want 3", len(contents)) + } + var replayed *genai.Part + for _, p := range contents[1].Parts { + if p.FunctionCall != nil { + replayed = p + } + } + if replayed == nil { + t.Fatal("the replayed assistant turn carries no functionCall") + } + if string(replayed.ThoughtSignature) != sig { + t.Fatalf("replayed signature = %q, want %q — a missing or altered one is a hard 400, "+ + "not a degraded response", replayed.ThoughtSignature, sig) + } +} + +// A tool_use block with no signature (one that came from another vendor, or a +// model that issued none) replays without inventing one. +func TestGeminiReplayWithoutSignatureSendsNone(t *testing.T) { + g := NewGemini("key") + contents := g.buildContents(wire.Request{Messages: []wire.Message{ + {Role: "assistant", Blocks: []wire.Block{ + {Type: "tool_use", ID: "tu_1", Name: "bash", Input: []byte(`{"cmd":"ls"}`)}, + }}, + }}) + for _, p := range contents[0].Parts { + if p.FunctionCall != nil && len(p.ThoughtSignature) != 0 { + t.Fatalf("invented a signature %q for a call that never had one", p.ThoughtSignature) + } + } +} From bfdaa5d8ca582b7a2b78a370f5333f93b007e5ee Mon Sep 17 00:00:00 2001 From: Tim Erwin Date: Thu, 3 Sep 2026 11:47:03 +0700 Subject: [PATCH 2/5] llm: request-shape errors are terminal, timing errors walk the chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 400 means WE malformed the request; a 401/403 means credentials; a 404 means the model does not exist. None of those change because a different model receives them, so walking the fallback chain just multiplies one bug into N identical failures — 298 failed calls in the broken review run came from exactly this. 408/429 and 5xx are timing, and still walk. Classifies by category rather than by individual status, with a table test pinning both halves. --- internal/llm/policy_test.go | 49 ++++++++++++++++++++++++++++++++--- internal/llm/recover.go | 51 ++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/internal/llm/policy_test.go b/internal/llm/policy_test.go index 4aa5e97..5294384 100644 --- a/internal/llm/policy_test.go +++ b/internal/llm/policy_test.go @@ -9,11 +9,13 @@ package llm import ( "context" "errors" + "fmt" "strings" "testing" "github.com/memcode-ai/memcode/internal/provider" "github.com/memcode-ai/memcode/internal/wire" + "google.golang.org/genai" ) // scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested @@ -206,9 +208,9 @@ func TestFallbackWalkOnModelError(t *testing.T) { if err != nil { t.Fatalf("the chain must rescue the call: %v", err) } - // glm-5p2 fails → catalog chain: kimi-k2p7-code. - if len(p.requested) != 2 || p.requested[1] != "kimi-k2p7-code" { - t.Fatalf("walk = %v, want [glm-5p2 kimi-k2p7-code]", p.requested) + // glm-5p2 fails → catalog chain: terra. + if len(p.requested) != 2 || p.requested[1] != "terra" { + t.Fatalf("walk = %v, want [glm-5p2 terra]", p.requested) } if !strings.HasPrefix(resp.FallbackReason, "model_error: ") { t.Fatalf("reason = %q, want model_error: …", resp.FallbackReason) @@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) { t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0]) } } + +// A 4xx that describes OUR request must never walk the fallback chain. The +// next model would receive the same malformed conversation and fail the same +// way — which is how one Gemini thought-signature 400 became 298 failed calls +// across two models before anything gave up. +// +// 408 and 429 are the deliberate exceptions: those are timing, not shape, and +// another model genuinely may serve them. +func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) { + apiErr := func(code int) error { + return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"}) + } + for _, tc := range []struct { + name string + code int + wantCall int // total provider calls: 1 = terminal, 2 = walked + }{ + {"400 malformed request", 400, 1}, + {"401 unauthorized", 401, 1}, + {"403 forbidden", 403, 1}, + {"404 unknown model", 404, 1}, + {"408 timeout", 408, 2}, + {"429 rate limited", 429, 2}, + {"500 provider down", 500, 2}, + {"503 unavailable", 503, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}} + r := pinnedRunner(p, prodInfo(nil), "glm-5p2") + _, _ = r.Complete(context.Background(), MainLoop, userReq("hi")) + if len(p.requested) != tc.wantCall { + verb := "walked the chain" + if tc.wantCall == 1 { + verb = "stopped at the first model" + } + t.Fatalf("%d → %d calls (%v), want %d — it should have %s", + tc.code, len(p.requested), p.requested, tc.wantCall, verb) + } + }) + } +} diff --git a/internal/llm/recover.go b/internal/llm/recover.go index 00df263..ab1520a 100644 --- a/internal/llm/recover.go +++ b/internal/llm/recover.go @@ -3,9 +3,11 @@ package llm import ( "context" "errors" + "net/http" "github.com/memcode-ai/memcode/catalog" "github.com/memcode-ai/memcode/internal/provider" + "github.com/memcode-ai/memcode/internal/providers/provcore" "github.com/memcode-ai/memcode/internal/wire" ) @@ -30,6 +32,17 @@ const maxModelFallbacks = 2 // terminalForFallback reports errors the fallback chain must never touch — // they carry their own recovery policy elsewhere. +// IsTerminal reports whether an error will fail identically no matter which +// model receives it — billing and auth state, or a request WE malformed. +// +// Callers outside the fallback walk need this too. A delegated worker that dies +// on a terminal error is not a retryable tool failure: handing "agent failed" +// back to the model invites it to try again, and it did — a Gemini +// thought-signature 400 was retried roughly 150 times over 12 minutes, because +// nothing in the loop could tell "this tool had a bad day" from "this will +// never work". +func IsTerminal(err error) bool { return err != nil && terminalForFallback(err) } + func terminalForFallback(err error) bool { for _, sentinel := range []error{ wire.ErrContextOverflow, // compact-and-retry (runLoop) @@ -52,7 +65,43 @@ func terminalForFallback(err error) bool { if errors.As(err, &exh) || errors.As(err, &noLane) { return true } - return errors.Is(err, provider.ErrNotLoggedIn) || errors.Is(err, provider.ErrGatewayOnly) + if errors.Is(err, provider.ErrNotLoggedIn) || errors.Is(err, provider.ErrGatewayOnly) { + return true + } + return terminalStatus(err) +} + +// terminalStatus classifies a provider's HTTP status. The rule the whole +// fallback chain rests on: +// +// The pinned model is authoritative. Fallbacks exist to survive +// INFRASTRUCTURE failure, not to reinterpret the request. +// +// A 4xx that describes OUR request is not an infrastructure failure. Retrying +// it on another model sends the same malformed conversation and fails the same +// way — which is exactly what happened when a Gemini thought-signature 400 was +// walked: one bug became 298 failed calls across two models before anything +// gave up. +// +// Deliberately NOT "all 4xx are terminal": 408 and 429 are timing, not shape, +// and a different model genuinely may serve them. +// +// 400 malformed / invalid argument -> terminal +// 401 / 403 auth -> terminal +// 404 unknown model or endpoint -> terminal (for this model) +// 408 timeout, 429 rate limit -> walk the chain +// 5xx, transport, timeouts -> walk the chain +// capability gap -> refused earlier, never reaches here +func terminalStatus(err error) bool { + code, _, ok := provcore.APIErrorInfo(err) + if !ok { + return false // not an HTTP-shaped error: a transport failure, which walks + } + switch code { + case http.StatusRequestTimeout, http.StatusTooManyRequests: + return false + } + return code >= 400 && code < 500 } // billingClass reports the errors that mean the org's billing state just From 58edc87318d29c49c4a13ab024a1fa175fe243ac Mon Sep 17 00:00:00 2001 From: Tim Erwin Date: Thu, 3 Sep 2026 11:47:24 +0700 Subject: [PATCH 3/5] catalog: fallbacks must leave the vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A same-vendor fallback is not a fallback. gemini-flash fell back to gemini-pro, which is the same adapter over the same API — so the thought-signature 400 hit the backup identically, and the chain existed only to fail twice as expensively. Repoints all 16 chains so the first hop always changes vendor (gemini-flash → haiku, luna; gemini-pro → sonnet, terra; glm-5p2 → terra, luna), and adds a catalog guard test so a same-vendor first hop cannot be reintroduced. Fallbacks exist to survive infrastructure failure, never to reinterpret the user's pin: a capability gap still refuses rather than substituting. --- catalog/catalog_policy_test.go | 31 ++++++++++++++++++++++++++++++ catalog/models.json | 35 +++++++++++++++++++++------------- internal/llm/delegated_test.go | 4 ++-- models.json | 35 +++++++++++++++++++++------------- 4 files changed, 77 insertions(+), 28 deletions(-) diff --git a/catalog/catalog_policy_test.go b/catalog/catalog_policy_test.go index 81d319f..8cef2db 100644 --- a/catalog/catalog_policy_test.go +++ b/catalog/catalog_policy_test.go @@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) { // TestTierAltitude is DELETED with TierAltitude: it named which rung of its // vendor's frontier/balanced/cheap triple a model occupied, which only mattered // for remapping an Automatic pick to the same "altitude" on another vendor. + +// A fallback must fail DIFFERENTLY from the thing it covers. +// +// A same-vendor first hop shares the provider, the adapter and the request +// semantics, so it survives only a single-model outage and multiplies every +// other kind of failure. That is not theoretical: gemini-flash used to fall +// back to gemini-pro, and when a bug in the shared Gemini adapter rejected +// every tool-using turn, the chain dutifully reproduced the same failure on the +// second model. +// +// Later hops may return to the same vendor — by then the independent one has +// already been tried. +func TestFallbackFirstHopLeavesTheVendor(t *testing.T) { + for _, m := range CatalogModels() { + fb := FallbackChain(m.Label) + if len(fb) == 0 { + continue + } + first, ok := LookupModel(fb[0]) + if !ok { + t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+ + "model that does not exist silently shortens the safety net", m.Label, fb[0]) + continue + } + if first.Vendor == m.Vendor { + t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+ + "the provider and adapter cannot cover an outage or a bug in either — point the "+ + "first hop at a different vendor.", m.Label, m.Vendor, fb[0]) + } + } +} diff --git a/catalog/models.json b/catalog/models.json index c3eef7c..c19e4f6 100644 --- a/catalog/models.json +++ b/catalog/models.json @@ -48,7 +48,7 @@ }, "max_output": 65536, "fallback": [ - "gemini-flash", + "sonnet", "terra" ] }, @@ -74,7 +74,7 @@ }, "max_output": 65536, "fallback": [ - "gemini-pro", + "haiku", "luna" ] }, @@ -100,7 +100,8 @@ }, "max_output": 65536, "fallback": [ - "gemini-flash" + "haiku", + "luna" ] }, { @@ -218,6 +219,7 @@ }, "max_output": 128000, "fallback": [ + "opus", "terra" ] }, @@ -243,6 +245,7 @@ }, "max_output": 128000, "fallback": [ + "sonnet", "luna" ] }, @@ -268,6 +271,7 @@ }, "max_output": 128000, "fallback": [ + "haiku", "terra" ] }, @@ -290,8 +294,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "luna" ] }, { @@ -316,8 +320,8 @@ }, "max_output": 32000, "fallback": [ - "glm-5p2", - "terra" + "terra", + "glm-5p2" ] }, { @@ -339,8 +343,8 @@ }, "max_output": 32000, "fallback": [ - "glm-5p2", - "terra" + "terra", + "glm-5p2" ] }, { @@ -362,8 +366,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "glm-5p2" ] }, { @@ -388,8 +392,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "glm-5p2" ] }, { @@ -403,6 +407,7 @@ "price_out": 0.6, "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.", "fallback": [ + "luna", "glm-5p2" ] }, @@ -415,6 +420,7 @@ "reasoning": true, "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.", "fallback": [ + "terra", "glm-5p2" ] }, @@ -437,6 +443,7 @@ }, "max_output": 32000, "fallback": [ + "terra", "glm-5p2" ], "price_in": 1.32, @@ -462,6 +469,7 @@ }, "max_output": 32000, "fallback": [ + "luna", "glm-5p2" ], "price_in": 0.22, @@ -477,6 +485,7 @@ "reasoning": true, "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).", "fallback": [ + "luna", "glm-5p2" ], "price_in": 0.3, diff --git a/internal/llm/delegated_test.go b/internal/llm/delegated_test.go index 29a7afd..f5ad9ed 100644 --- a/internal/llm/delegated_test.go +++ b/internal/llm/delegated_test.go @@ -68,7 +68,7 @@ func TestDelegatedWorkerKeepsNormalFailureSemantics(t *testing.T) { if _, err := worker2.Complete(context.Background(), Agent, userReq("work")); err != nil { t.Fatalf("the declared chain must rescue a delegated worker too: %v", err) } - if len(p2.requested) != 2 || p2.requested[1] != "kimi-k2p7-code" { - t.Fatalf("delegated fallback walk = %v, want [glm-5p2 kimi-k2p7-code]", p2.requested) + if len(p2.requested) != 2 || p2.requested[1] != "terra" { + t.Fatalf("delegated fallback walk = %v, want [glm-5p2 terra]", p2.requested) } } diff --git a/models.json b/models.json index c3eef7c..c19e4f6 100644 --- a/models.json +++ b/models.json @@ -48,7 +48,7 @@ }, "max_output": 65536, "fallback": [ - "gemini-flash", + "sonnet", "terra" ] }, @@ -74,7 +74,7 @@ }, "max_output": 65536, "fallback": [ - "gemini-pro", + "haiku", "luna" ] }, @@ -100,7 +100,8 @@ }, "max_output": 65536, "fallback": [ - "gemini-flash" + "haiku", + "luna" ] }, { @@ -218,6 +219,7 @@ }, "max_output": 128000, "fallback": [ + "opus", "terra" ] }, @@ -243,6 +245,7 @@ }, "max_output": 128000, "fallback": [ + "sonnet", "luna" ] }, @@ -268,6 +271,7 @@ }, "max_output": 128000, "fallback": [ + "haiku", "terra" ] }, @@ -290,8 +294,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "luna" ] }, { @@ -316,8 +320,8 @@ }, "max_output": 32000, "fallback": [ - "glm-5p2", - "terra" + "terra", + "glm-5p2" ] }, { @@ -339,8 +343,8 @@ }, "max_output": 32000, "fallback": [ - "glm-5p2", - "terra" + "terra", + "glm-5p2" ] }, { @@ -362,8 +366,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "glm-5p2" ] }, { @@ -388,8 +392,8 @@ }, "max_output": 32000, "fallback": [ - "kimi-k2p7-code", - "terra" + "terra", + "glm-5p2" ] }, { @@ -403,6 +407,7 @@ "price_out": 0.6, "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.", "fallback": [ + "luna", "glm-5p2" ] }, @@ -415,6 +420,7 @@ "reasoning": true, "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.", "fallback": [ + "terra", "glm-5p2" ] }, @@ -437,6 +443,7 @@ }, "max_output": 32000, "fallback": [ + "terra", "glm-5p2" ], "price_in": 1.32, @@ -462,6 +469,7 @@ }, "max_output": 32000, "fallback": [ + "luna", "glm-5p2" ], "price_in": 0.22, @@ -477,6 +485,7 @@ "reasoning": true, "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).", "fallback": [ + "luna", "glm-5p2" ], "price_in": 0.3, From b859702d5d9e573416cdf6ae97d5e481d65c6a20 Mon Sep 17 00:00:00 2001 From: Tim Erwin Date: Thu, 3 Sep 2026 11:47:24 +0700 Subject: [PATCH 4/5] runtime: a terminal worker failure ends the turn with its cause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delegated worker whose model call failed came back to the model as errResult("agent failed: …"), an ordinary retryable tool error. For a terminal cause that is a lie: it will fail identically next time. The model read it as "try again" and did — the broken review run made ~150 identical failed calls, one every two seconds for 12 minutes, and only ended on the pipeline's own timeout. The real cause never appeared anywhere the user could see it. Terminal worker errors now arm turn.fatalErr, the loop ends the turn on it, and the existing path carries it the rest of the way: printed to the session, stored as LastError, returned as a non-zero exit and a failed gateway job. Retryable causes (429, 5xx, transport) are untouched and still just a tool error. Exports llm.IsTerminal for the classification, and pins both sides. --- internal/agent/runtime/exec.go | 23 +++++- internal/agent/runtime/fatalworker_test.go | 87 ++++++++++++++++++++++ internal/agent/runtime/loop.go | 12 +++ internal/agent/runtime/turnstate.go | 9 ++- 4 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 internal/agent/runtime/fatalworker_test.go diff --git a/internal/agent/runtime/exec.go b/internal/agent/runtime/exec.go index f6c7a4b..70335b5 100644 --- a/internal/agent/runtime/exec.go +++ b/internal/agent/runtime/exec.go @@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe if err != nil { s.toolLine(true, "Explore", scope, "failed", true) s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200))) - return errResult("explore failed: " + err.Error()) + return s.workerFailed("explore", err) } status := fmt.Sprintf("%d tools", res.ToolCalls) if res.ServedBy != "" { @@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string { // strong Anthropic), read-only or mutating, run it to completion, and return its result to the // calling model. This is the unified primitive — explore is its read-only-research sugar, and // (Phase 2) background runs reuse it via the jobs registry. + +// workerFailed reports a delegated worker's failure back to the model, and — when +// the cause is terminal (auth, billing, or a request WE malformed) — also arms the +// turn's fatal error so the loop stops instead of letting the model retry. +// +// A worker that died on a 400 will die on the next 400 identically. Handed back as +// a plain tool error it reads as "that had a bad day, try again", and the model +// does, forever: a dropped Gemini thought-signature produced ~150 identical failed +// calls over 12 minutes before the run gave up on a timeout, with the real cause +// never surfacing anywhere the user could see it. +func (s *Session) workerFailed(kind string, err error) toolResult { + if llm.IsTerminal(err) && s.turn.fatalErr == nil { + s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err) + } + return errResult(kind + " failed: " + err.Error()) +} + func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult { var in tools.AgentInput if err := json.Unmarshal(input, &in); err != nil { @@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu if err != nil { s.toolLine(true, "Agent", clip(task, 60), "failed", true) s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200))) - return errResult("agent failed: " + err.Error()) + return s.workerFailed("agent", err) } status := fmt.Sprintf("%d tools", res.ToolCalls) if res.ServedBy != "" { @@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR if err != nil { s.toolLine(true, "Dispatch", clip(task, 60), "failed", true) s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200))) - return errResult("dispatch failed: " + err.Error()) + return s.workerFailed("dispatch", err) } s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false) return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+ diff --git a/internal/agent/runtime/fatalworker_test.go b/internal/agent/runtime/fatalworker_test.go new file mode 100644 index 0000000..4b512a7 --- /dev/null +++ b/internal/agent/runtime/fatalworker_test.go @@ -0,0 +1,87 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "testing" + + "github.com/memcode-ai/memcode/internal/agent/permissions" + "github.com/memcode-ai/memcode/internal/store" + "github.com/memcode-ai/memcode/internal/wire" + "google.golang.org/genai" +) + +// apiErr builds a real provider API error, the way llm's fallback tests do — the +// terminal/retryable split is read off the HTTP status by the registered extractor. +func apiErr(code int, msg string) error { + return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg}) +} + +// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini +// thought-signature bug, which fails identically no matter how often it is retried. +type fatalWorkerProvider struct{ calls int } + +func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) { + p.calls++ + return wire.Response{}, apiErr(400, "function call missing thoughtSignature") +} + +// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a +// terminal error must arm turn.fatalErr — the loop reads that and ends the turn +// with the cause — rather than handing the model a retryable "agent failed". +// +// This is the 12-minute review-run regression: ~150 identical 400s, one every two +// seconds, because nothing could distinguish "try again" from "this can never work". +func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard) + s.turn = newTurnState() + + res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature")) + + if s.turn.fatalErr == nil { + t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying") + } + // The cause survives to the surface: the loop returns this error, and the + // runtime prints it — the user sees "gemini 400", not a silent 20-minute hang. + if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") { + t.Errorf("fatal error must name the worker AND the cause, got %q", got) + } + // The model still gets told, so a turn that somehow continues isn't left blind. + if len(res.blocks) == 0 { + t.Error("the tool result must still be returned to the model") + } +} + +// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network +// blip IS worth retrying, so it stays an ordinary tool error and the turn lives. +func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard) + + for _, err := range []error{ + apiErr(429, "rate limited"), + apiErr(503, "backend unavailable"), + errors.New("connection reset by peer"), + } { + s.turn = newTurnState() + s.workerFailed("explore", err) + if s.turn.fatalErr != nil { + t.Errorf("%v is retryable and must NOT kill the turn", err) + } + } +} diff --git a/internal/agent/runtime/loop.go b/internal/agent/runtime/loop.go index 1795b64..287a80d 100644 --- a/internal/agent/runtime/loop.go +++ b/internal/agent/runtime/loop.go @@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire. *messages = append(*messages, wire.Message{Role: "user", Blocks: results}) s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions + // A delegated worker hit a terminal failure — it cannot succeed on a retry, + // so end the turn with the real cause instead of looping on it. + if s.turn.fatalErr != nil { + return iterations, false, s.turn.fatalErr + } + // execute_plan fired this batch: ExitPlan just flipped the state machine into the // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run @@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir results := s.executeBatchHooked(ctx, uses) *messages = append(*messages, wire.Message{Role: "user", Blocks: results}) s.recordTurn(resp.Text(), uses) + + // A delegated worker hit a terminal failure — it cannot succeed on a retry, + // so end the turn with the real cause instead of looping on it. + if s.turn.fatalErr != nil { + return resp, "", s.turn.fatalErr + } } return resp, strings.TrimSpace(resp.Text()), nil } diff --git a/internal/agent/runtime/turnstate.go b/internal/agent/runtime/turnstate.go index 4f49d42..056871d 100644 --- a/internal/agent/runtime/turnstate.go +++ b/internal/agent/runtime/turnstate.go @@ -20,8 +20,13 @@ type turnState struct { redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation lessonDone bool // a lesson was already distilled this turn (fire once) - billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure - laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane + // fatalErr is a terminal failure raised from INSIDE a tool — a delegated + // worker whose model call cannot succeed on any retry. It aborts the turn + // after the batch, with the real cause, instead of being handed back as a + // tool error the model will cheerfully retry forever. + fatalErr error + billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure + laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane } // newTurnState returns a fresh per-turn state (with an initialized gather tracker). From e8bcb28cfcec44aefbb0d9599ac54318eec478e9 Mon Sep 17 00:00:00 2001 From: Tim Erwin Date: Thu, 3 Sep 2026 12:06:34 +0700 Subject: [PATCH 5/5] compat: carry the tool-call signature across the hosted wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixing the Gemini adapter was necessary but not sufficient. On the hosted path the CLI never touches that adapter: the tool call crosses the wire in OpenAI-compat shape, which has nowhere to put opaque per-call provider state, so the signature was dropped in translation and every replay 400'd regardless. It rides a namespaced memcode_signature extension now — the same channel pattern as memcode_opaque for reasoning blocks — through all four hops: streaming delta, accumulator, block assembly, and the encode back on replay. omitempty keeps the standard shape standard for providers that issue no signature. Verified live end to end against the deployed gateway: three sequential tool rounds on gemini-flash, correct answers, no retries and no fallback. The same prompt before this change failed three times and fell over to haiku. --- internal/providers/compat/decode.go | 9 ++- internal/providers/compat/encode.go | 4 +- internal/providers/compat/signature_test.go | 79 +++++++++++++++++++++ internal/providers/compat/wire.go | 12 ++++ 4 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 internal/providers/compat/signature_test.go diff --git a/internal/providers/compat/decode.go b/internal/providers/compat/decode.go index f9c185c..651711e 100644 --- a/internal/providers/compat/decode.go +++ b/internal/providers/compat/decode.go @@ -69,7 +69,7 @@ func blocksFrom(opaque []json.RawMessage, text string, calls []ToolCall, memcode args = "{}" } blocks = append(blocks, wire.Block{Type: "tool_use", ID: tc.ID, Name: tc.Function.Name, - Input: json.RawMessage(args)}) + Input: json.RawMessage(args), Signature: tc.MemcodeSignature}) } return blocks } @@ -129,6 +129,7 @@ func applyExt(resp *wire.Response, ext *MemcodeExt, memcode bool) { // arguments; both assemble identically). type callAccum struct { id string + sig string // opaque per-call provider state (Gemini thoughtSignature) name, args strings.Builder } @@ -184,6 +185,9 @@ func (a *streamAccum) apply(c ChatChunk, h wire.StreamHandler) { if td.ID != "" { ca.id = td.ID } + if td.MemcodeSignature != "" { + ca.sig = td.MemcodeSignature + } if td.Function != nil { ca.name.WriteString(td.Function.Name) ca.args.WriteString(td.Function.Arguments) @@ -215,7 +219,8 @@ func (a *streamAccum) response() wire.Response { for _, i := range a.order { ca := a.calls[i] calls = append(calls, ToolCall{ID: ca.id, Type: "function", - Function: FunctionCall{Name: ca.name.String(), Arguments: ca.args.String()}}) + Function: FunctionCall{Name: ca.name.String(), Arguments: ca.args.String()}, + MemcodeSignature: ca.sig}) } resp := wire.Response{Model: a.model, Backend: backendFor(a.memcode), StopReason: stopReasonFrom(a.finish)} resp.Blocks = blocksFrom(a.opaque, a.text.String(), calls, a.memcode) diff --git a/internal/providers/compat/encode.go b/internal/providers/compat/encode.go index 285fdc5..688f618 100644 --- a/internal/providers/compat/encode.go +++ b/internal/providers/compat/encode.go @@ -221,7 +221,9 @@ func encodeAssistant(m wire.Message, memcode bool) (ChatMessage, error) { args = "{}" } out.ToolCalls = append(out.ToolCalls, ToolCall{ID: b.ID, Type: "function", - Function: FunctionCall{Name: b.Name, Arguments: args}}) + Function: FunctionCall{Name: b.Name, Arguments: args}, + // Handed back verbatim: without it Gemini rejects the replay. + MemcodeSignature: b.Signature}) default: return out, fmt.Errorf("unsupported block type %q", b.Type) } diff --git a/internal/providers/compat/signature_test.go b/internal/providers/compat/signature_test.go new file mode 100644 index 0000000..228e7dd --- /dev/null +++ b/internal/providers/compat/signature_test.go @@ -0,0 +1,79 @@ +package compat + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/memcode-ai/memcode/internal/wire" +) + +// TestToolCallSignatureSurvivesTheHostedRoundTrip is the production half of the +// Gemini thought-signature bug. Fixing the Gemini ADAPTER was not enough: on the +// hosted path the CLI never touches that adapter, and the tool call crosses the +// wire in OpenAI-compat shape, which has no field for opaque per-call state. The +// signature was dropped in translation, so every replay 400'd anyway. +// +// This pins the whole path: stream delta → block → encoded back for replay. +func TestToolCallSignatureSurvivesTheHostedRoundTrip(t *testing.T) { + const sig = "opaque-thought-signature-from-gemini" + + // Inbound: the gateway streams the call with its signature attached. + a := newStreamAccum(true) + fn := FunctionCall{Name: "ripgrep", Arguments: `{"pattern":"func"}`} + a.apply(ChatChunk{Choices: []ChunkChoice{{Delta: Delta{ + ToolCalls: []ToolCallDelta{{Index: 0, ID: "call_1", Type: "function", Function: &fn, MemcodeSignature: sig}}, + }}}}, wire.StreamHandler{}) + resp := a.response() + + var use wire.Block + for _, b := range resp.Blocks { + if b.Type == "tool_use" { + use = b + } + } + if use.ID != "call_1" { + t.Fatalf("no tool_use decoded from the stream: %+v", resp.Blocks) + } + if use.Signature != sig { + t.Fatalf("signature lost decoding the stream: got %q want %q", use.Signature, sig) + } + + // Outbound: replaying that block must carry the signature back, or Gemini + // rejects the turn with "Function call is missing a thought_signature". + msg, err := encodeAssistant(wire.Message{Role: "assistant", Blocks: []wire.Block{use}}, true) + if err != nil { + t.Fatal(err) + } + if len(msg.ToolCalls) != 1 || msg.ToolCalls[0].MemcodeSignature != sig { + t.Fatalf("signature lost encoding the replay: %+v", msg.ToolCalls) + } + + // And it must actually reach the wire under its namespaced key. + raw, err := json.Marshal(msg) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"memcode_signature":"`+sig+`"`) { + t.Fatalf("signature is not on the wire: %s", raw) + } +} + +// TestToolCallWithoutSignatureStaysClean: providers that issue no signature must +// not gain an empty field — omitempty keeps the standard shape standard, so a +// non-memcode OpenAI-compat server sees exactly what it expects. +func TestToolCallWithoutSignatureStaysClean(t *testing.T) { + msg, err := encodeAssistant(wire.Message{Role: "assistant", Blocks: []wire.Block{ + {Type: "tool_use", ID: "call_1", Name: "ripgrep", Input: json.RawMessage(`{}`)}, + }}, true) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(msg) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "memcode_signature") { + t.Fatalf("empty signature must not appear on the wire: %s", raw) + } +} diff --git a/internal/providers/compat/wire.go b/internal/providers/compat/wire.go index 600072d..2ccda7c 100644 --- a/internal/providers/compat/wire.go +++ b/internal/providers/compat/wire.go @@ -205,6 +205,14 @@ type ToolCall struct { ID string `json:"id"` Type string `json:"type"` // "function" Function FunctionCall `json:"function"` + + // MemcodeSignature carries opaque provider state that belongs to THIS call + // and must come back verbatim when the call is replayed — Gemini issues a + // thoughtSignature with every functionCall and rejects the replay with a 400 + // without it. The standard tool_calls shape has nowhere to put that, so it + // rides a namespaced extension, the same way reasoning blocks ride + // memcode_opaque. Ignored by any server that does not know it. + MemcodeSignature string `json:"memcode_signature,omitempty"` } // FunctionCall is a call's name + JSON-encoded arguments string. @@ -307,6 +315,10 @@ type ToolCallDelta struct { ID string `json:"id,omitempty"` Type string `json:"type,omitempty"` Function *FunctionCall `json:"function,omitempty"` + + // MemcodeSignature is the streaming half of ToolCall.MemcodeSignature — + // sent once on the delta that opens the call. + MemcodeSignature string `json:"memcode_signature,omitempty"` } // ── models + errors ─────────────────────────────────────────────────────────