Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timothyerwin merged 6 commits into from
Sep 3, 2026
Merged

Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
efe9b3b
gemini: round-trip thoughtSignature through tool replay
timothyerwin Sep 3, 2026
bfdaa5d
llm: request-shape errors are terminal, timing errors walk the chain
timothyerwin Sep 3, 2026
58edc87
catalog: fallbacks must leave the vendor
timothyerwin Sep 3, 2026
b859702
runtime: a terminal worker failure ends the turn with its cause
timothyerwin Sep 3, 2026
e8bcb28
compat: carry the tool-call signature across the hosted wire
timothyerwin Sep 3, 2026
8b7d734
merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks
timothyerwin Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions catalog/catalog_policy_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
for _, m := range CatalogModels() {
fb := FallbackChain(m.Label)
if len(fb) == 0 {
continue
}
first, ok := LookupModel(fb[0])
if !ok {
t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
"model that does not exist silently shortens the safety net", m.Label, fb[0])
continue
}
if first.Vendor == m.Vendor {
t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
"the provider and adapter cannot cover an outage or a bug in either — point the "+
"first hop at a different vendor.", m.Label, m.Vendor, fb[0])
}
}
}
26 changes: 18 additions & 8 deletions catalog/models.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
},
"max_output": 65536,
"fallback": [
"gemini-flash",
"sonnet",
"terra"
]
},
Expand All@@ -74,7 +74,7 @@
},
"max_output": 65536,
"fallback": [
"gemini-pro",
"haiku",
"luna"
]
},
Expand All@@ -100,7 +100,8 @@
},
"max_output": 65536,
"fallback": [
"gemini-flash"
"haiku",
"luna"
]
},
{
Expand DownExpand Up@@ -218,6 +219,7 @@
},
"max_output": 128000,
"fallback": [
"opus",
"terra"
]
},
Expand All@@ -243,6 +245,7 @@
},
"max_output": 128000,
"fallback": [
"sonnet",
"luna"
]
},
Expand All@@ -268,6 +271,7 @@
},
"max_output": 128000,
"fallback": [
"haiku",
"terra"
]
},
Expand All@@ -290,7 +294,8 @@
},
"max_output": 32000,
"fallback": [
"terra"
"terra",
"luna"
]
},
{
Expand All@@ -315,8 +320,8 @@
},
"max_output": 32000,
"fallback": [
"glm-5p2",
"terra"
"terra",
"glm-5p2"
]
},
{
Expand All@@ -341,8 +346,8 @@
},
"max_output": 32000,
"fallback": [
"glm-5p2",
"terra"
"terra",
"glm-5p2"
]
},
{
Expand All@@ -356,6 +361,7 @@
"price_out": 0.6,
"_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
"fallback": [
"luna",
"glm-5p2"
]
},
Expand All@@ -368,6 +374,7 @@
"reasoning": true,
"_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
"fallback": [
"terra",
"glm-5p2"
]
},
Expand All@@ -390,6 +397,7 @@
},
"max_output": 32000,
"fallback": [
"terra",
"glm-5p2"
],
"price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
},
"max_output": 32000,
"fallback": [
"luna",
"glm-5p2"
],
"price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
"reasoning": true,
"_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
"fallback": [
"luna",
"glm-5p2"
],
"price_in": 0.3,
Expand Down
23 changes: 20 additions & 3 deletions internal/agent/runtime/exec.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
if err != nil {
s.toolLine(true, "Explore", scope, "failed", true)
s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
return errResult("explore failed: " + err.Error())
return s.workerFailed("explore", err)
}
status := fmt.Sprintf("%d tools", res.ToolCalls)
if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
if llm.IsTerminal(err) && s.turn.fatalErr == nil {
s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
}
return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
var in tools.AgentInput
if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
if err != nil {
s.toolLine(true, "Agent", clip(task, 60), "failed", true)
s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
return errResult("agent failed: " + err.Error())
return s.workerFailed("agent", err)
}
status := fmt.Sprintf("%d tools", res.ToolCalls)
if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
if err != nil {
s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
return errResult("dispatch failed: " + err.Error())
return s.workerFailed("dispatch", err)
}
s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down
87 changes: 87 additions & 0 deletions internal/agent/runtime/fatalworker_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
"context"
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"testing"

"github.com/memcode-ai/memcode/internal/agent/permissions"
"github.com/memcode-ai/memcode/internal/store"
"github.com/memcode-ai/memcode/internal/wire"
"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
p.calls++
return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
ctx := context.Background()
st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
s.turn = newTurnState()

res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

if s.turn.fatalErr == nil {
t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
}
// The cause survives to the surface: the loop returns this error, and the
// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
t.Errorf("fatal error must name the worker AND the cause, got %q", got)
}
// The model still gets told, so a turn that somehow continues isn't left blind.
if len(res.blocks) == 0 {
t.Error("the tool result must still be returned to the model")
}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
ctx := context.Background()
st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

for _, err := range []error{
apiErr(429, "rate limited"),
apiErr(503, "backend unavailable"),
errors.New("connection reset by peer"),
} {
s.turn = newTurnState()
s.workerFailed("explore", err)
if s.turn.fatalErr != nil {
t.Errorf("%v is retryable and must NOT kill the turn", err)
}
}
}
12 changes: 12 additions & 0 deletions internal/agent/runtime/loop.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
*messages = append(*messages, wire.Message{Role: "user", Blocks: results})
s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

// A delegated worker hit a terminal failure — it cannot succeed on a retry,
// so end the turn with the real cause instead of looping on it.
if s.turn.fatalErr != nil {
return iterations, false, s.turn.fatalErr
}

// execute_plan fired this batch: ExitPlan just flipped the state machine into the
// apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
// apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
results := s.executeBatchHooked(ctx, uses)
*messages = append(*messages, wire.Message{Role: "user", Blocks: results})
s.recordTurn(resp.Text(), uses)

// A delegated worker hit a terminal failure — it cannot succeed on a retry,
// so end the turn with the real cause instead of looping on it.
if s.turn.fatalErr != nil {
return resp, "", s.turn.fatalErr
}
}
return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down
9 changes: 7 additions & 2 deletions internal/agent/runtime/turnstate.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
lessonDone bool // a lesson was already distilled this turn (fire once)
billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
// worker whose model call cannot succeed on any retry. It aborts the turn
// after the batch, with the real cause, instead of being handed back as a
// tool error the model will cheerfully retry forever.
fatalErr error
billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down
43 changes: 43 additions & 0 deletions internal/llm/policy_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
"context"
"errors"
"fmt"
"strings"
"testing"

"github.com/memcode-ai/memcode/internal/provider"
"github.com/memcode-ai/memcode/internal/wire"
"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
apiErr := func(code int) error {
return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
}
for _, tc := range []struct {
name string
code int
wantCall int // total provider calls: 1 = terminal, 2 = walked
}{
{"400 malformed request", 400, 1},
{"401 unauthorized", 401, 1},
{"403 forbidden", 403, 1},
{"404 unknown model", 404, 1},
{"408 timeout", 408, 2},
{"429 rate limited", 429, 2},
{"500 provider down", 500, 2},
{"503 unavailable", 503, 2},
} {
t.Run(tc.name, func(t *testing.T) {
p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
_, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
if len(p.requested) != tc.wantCall {
verb := "walked the chain"
if tc.wantCall == 1 {
verb = "stopped at the first model"
}
t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
tc.code, len(p.requested), p.requested, tc.wantCall, verb)
}
})
}
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

timothyerwin
merged 6 commits into

from

Sep 3, 2026
 Merged


Fix the Gemini tool-replay 400, and stop it from costing 12 minutes of silence
#8
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
efe9b3b

gemini: round-trip thoughtSignature through tool replay


timothyerwin Sep 3, 2026
bfdaa5d

llm: request-shape errors are terminal, timing errors walk the chain


timothyerwin Sep 3, 2026
58edc87

catalog: fallbacks must leave the vendor


timothyerwin Sep 3, 2026
b859702

runtime: a terminal worker failure ends the turn with its cause


timothyerwin Sep 3, 2026
e8bcb28

compat: carry the tool-call signature across the hosted wire


timothyerwin Sep 3, 2026
8b7d734

merge main: keep #7's Kimi retirement, keep the cross-vendor fallbacks


timothyerwin Sep 3, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
catalog/catalog_policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,3 +65,34 @@ func TestCatalogFallbackChainsResolve(t *testing.T) {
// TestTierAltitude is DELETED with TierAltitude: it named which rung of its
// vendor's frontier/balanced/cheap triple a model occupied, which only mattered
// for remapping an Automatic pick to the same "altitude" on another vendor.

// A fallback must fail DIFFERENTLY from the thing it covers.
//
// A same-vendor first hop shares the provider, the adapter and the request
// semantics, so it survives only a single-model outage and multiplies every
// other kind of failure. That is not theoretical: gemini-flash used to fall
// back to gemini-pro, and when a bug in the shared Gemini adapter rejected
// every tool-using turn, the chain dutifully reproduced the same failure on the
// second model.
//
// Later hops may return to the same vendor — by then the independent one has
// already been tried.
func TestFallbackFirstHopLeavesTheVendor(t *testing.T) {
	for _, m := range CatalogModels() {
 fb := FallbackChain(m.Label)
 if len(fb) == 0 {
 continue
 }
 first, ok := LookupModel(fb[0])
 if !ok {
 t.Errorf("%s falls back to %q, which is not in the catalog — a chain naming a "+
 "model that does not exist silently shortens the safety net", m.Label, fb[0])
 continue
 }
 if first.Vendor == m.Vendor {
 t.Errorf("%s (%s) falls back first to %s, the SAME vendor. A fallback that shares "+
 "the provider and adapter cannot cover an outage or a bug in either — point the "+
 "first hop at a different vendor.", m.Label, m.Vendor, fb[0])
 }
	}
}

26 changes: 18 additions & 8 deletions
catalog/models.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash",
 "sonnet",
 "terra"
 ]
 },
Expand All@@ -74,7 +74,7 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-pro",
 "haiku",
 "luna"
 ]
 },
Expand All@@ -100,7 +100,8 @@
 },
 "max_output": 65536,
 "fallback": [
 "gemini-flash"
 "haiku",
 "luna"
 ]
 },
 {
Expand DownExpand Up@@ -218,6 +219,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "opus",
 "terra"
 ]
 },
Expand All@@ -243,6 +245,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "sonnet",
 "luna"
 ]
 },
Expand All@@ -268,6 +271,7 @@
 },
 "max_output": 128000,
 "fallback": [
 "haiku",
 "terra"
 ]
 },
Expand All@@ -290,7 +294,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra"
 "terra",
 "luna"
 ]
 },
 {
Expand All@@ -315,8 +320,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -341,8 +346,8 @@
 },
 "max_output": 32000,
 "fallback": [
 "glm-5p2",
 "terra"
 "terra",
 "glm-5p2"
 ]
 },
 {
Expand All@@ -356,6 +361,7 @@
 "price_out": 0.6,
 "_note": "OpenAI gpt-oss-120b (open-weight, Fireworks serverless). CLASSIFY role 2026-06-21: the cheap structured-output classifier for the followup-fold queue + plan-intent + risk/authorize judges. LIVE-PROBED against glm-5p2 on a followup-fold battery: gpt-oss-120b 10/10 @1.6-2.6s; glm-5p2 only 7/10 (biased to 'false' \u2192 under-folds) at ~10x the token cost. Forces tools cleanly (tool_choice). price ~$0.15/$0.60 per M (approx). The cheap CODING lane stays glm-5p2; this is classify-only. Rollback: drop config.json roles.classify (falls back to the cheap lane) + redeploy.",
 "fallback": [
 "luna",
 "glm-5p2"
 ]
 },
Expand All@@ -368,6 +374,7 @@
 "reasoning": true,
 "_note": "GLM-5.1 (Z.ai): prior planner + cheap lane, SUPERSEDED by glm-5p2 2026-06-17. Kept as the instant rollback target. Function-calling, 202K, no vision. Fallback id accounts/fireworks/models/glm-5.",
 "fallback": [
 "terra",
 "glm-5p2"
 ]
 },
Expand All@@ -390,6 +397,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "terra",
 "glm-5p2"
 ],
 "price_in": 1.32,
Expand All@@ -415,6 +423,7 @@
 },
 "max_output": 32000,
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.22,
Expand All@@ -430,6 +439,7 @@
 "reasoning": true,
 "_note": "RETIRED from roles 2026-06-13 \u2014 recurring tool-call-text leak (opencode #30684 + cross-harness) + weak instruction-following. Kept for reference; not assigned to any role. PRICING verified 2026-08-15 fireworks.ai model page: $0.30/$1.20, cached $0.059 (was billing the glm-family fallback $1.40/$4.40).",
 "fallback": [
 "luna",
 "glm-5p2"
 ],
 "price_in": 0.3,
Expand Down

23 changes: 20 additions & 3 deletions
internal/agent/runtime/exec.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -484,7 +484,7 @@ func (s *Session) exploreTool(ctx context.Context, input json.RawMessage) toolRe
	if err != nil {
 s.toolLine(true, "Explore", scope, "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("explore failed: " + err.Error())
 return s.workerFailed("explore", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -556,6 +556,23 @@ func sanitizeReportKind(kind string) string {
// strong Anthropic), read-only or mutating, run it to completion, and return its result to the
// calling model. This is the unified primitive — explore is its read-only-research sugar, and
// (Phase 2) background runs reuse it via the jobs registry.

// workerFailed reports a delegated worker's failure back to the model, and — when
// the cause is terminal (auth, billing, or a request WE malformed) — also arms the
// turn's fatal error so the loop stops instead of letting the model retry.
//
// A worker that died on a 400 will die on the next 400 identically. Handed back as
// a plain tool error it reads as "that had a bad day, try again", and the model
// does, forever: a dropped Gemini thought-signature produced ~150 identical failed
// calls over 12 minutes before the run gave up on a timeout, with the real cause
// never surfacing anywhere the user could see it.
func (s *Session) workerFailed(kind string, err error) toolResult {
	if llm.IsTerminal(err) && s.turn.fatalErr == nil {
 s.turn.fatalErr = fmt.Errorf("%s: %w", kind, err)
	}
	return errResult(kind + " failed: " + err.Error())
}

func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResult {
	var in tools.AgentInput
	if err := json.Unmarshal(input, &in); err != nil {
Expand DownExpand Up@@ -588,7 +605,7 @@ func (s *Session) agentTool(ctx context.Context, input json.RawMessage) toolResu
	if err != nil {
 s.toolLine(true, "Agent", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("agent failed: " + err.Error())
 return s.workerFailed("agent", err)
	}
	status := fmt.Sprintf("%d tools", res.ToolCalls)
	if res.ServedBy != "" {
Expand DownExpand Up@@ -642,7 +659,7 @@ func (s *Session) dispatchTool(ctx context.Context, input json.RawMessage) toolR
	if err != nil {
 s.toolLine(true, "Dispatch", clip(task, 60), "failed", true)
 s.printf("%s\n", metaStyle.Render(" ⎿ failed: "+clip(err.Error(), 200)))
 return errResult("dispatch failed: " + err.Error())
 return s.workerFailed("dispatch", err)
	}
	s.toolLine(true, "Dispatch", clip(task, 60), fmt.Sprintf("started %s (pid %d)", job.ID, job.PID), false)
	return textResult(fmt.Sprintf("dispatched sub-agent %s (pid %d) — running hands-off in %s mode.\n"+
Expand Down

87 changes: 87 additions & 0 deletions
internal/agent/runtime/fatalworker_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
package runtime

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/agent/permissions"
	"github.com/memcode-ai/memcode/internal/store"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// apiErr builds a real provider API error, the way llm's fallback tests do — the
// terminal/retryable split is read off the HTTP status by the registered extractor.
func apiErr(code int, msg string) error {
	return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: msg})
}

// fatalWorkerProvider fails EVERY call with a 400 — the shape of the Gemini
// thought-signature bug, which fails identically no matter how often it is retried.
type fatalWorkerProvider struct{ calls int }

func (p *fatalWorkerProvider) Complete(context.Context, wire.Request) (wire.Response, error) {
	p.calls++
	return wire.Response{}, apiErr(400, "function call missing thoughtSignature")
}

// TestTerminalWorkerErrorArmsTheTurnFatal: a delegated worker that dies on a
// terminal error must arm turn.fatalErr — the loop reads that and ends the turn
// with the cause — rather than handing the model a retryable "agent failed".
//
// This is the 12-minute review-run regression: ~150 identical 400s, one every two
// seconds, because nothing could distinguish "try again" from "this can never work".
func TestTerminalWorkerErrorArmsTheTurnFatal(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)
	s.turn = newTurnState()

	res := s.workerFailed("agent", apiErr(400, "function call missing thoughtSignature"))

	if s.turn.fatalErr == nil {
 t.Fatal("a terminal worker error must arm turn.fatalErr so the loop stops retrying")
	}
	// The cause survives to the surface: the loop returns this error, and the
	// runtime prints it — the user sees "gemini 400", not a silent 20-minute hang.
	if got := s.turn.fatalErr.Error(); !strings.Contains(got, "thoughtSignature") || !strings.Contains(got, "agent") {
 t.Errorf("fatal error must name the worker AND the cause, got %q", got)
	}
	// The model still gets told, so a turn that somehow continues isn't left blind.
	if len(res.blocks) == 0 {
 t.Error("the tool result must still be returned to the model")
	}
}

// TestTransientWorkerErrorDoesNotKillTheTurn: the flip side. A 429 or a network
// blip IS worth retrying, so it stays an ordinary tool error and the turn lives.
func TestTransientWorkerErrorDoesNotKillTheTurn(t *testing.T) {
	ctx := context.Background()
	st, err := store.Open(ctx, filepath.Join(t.TempDir(), "state.db"))
	if err != nil {
 t.Fatal(err)
	}
	defer st.Close()
	s := newSess(st, &fatalWorkerProvider{}, t.TempDir(), "sonnet", permissions.ModeAsk, io.Discard)

	for _, err := range []error{
 apiErr(429, "rate limited"),
 apiErr(503, "backend unavailable"),
 errors.New("connection reset by peer"),
	} {
 s.turn = newTurnState()
 s.workerFailed("explore", err)
 if s.turn.fatalErr != nil {
 t.Errorf("%v is retryable and must NOT kill the turn", err)
 }
	}
}

12 changes: 12 additions & 0 deletions
internal/agent/runtime/loop.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -456,6 +456,12 @@ func (s *Session) runLoop(ctx context.Context, sys promptSpec, messages *[]wire.
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses) // episodic log: assistant text + meaningful actions

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return iterations, false, s.turn.fatalErr
 }

 // execute_plan fired this batch: ExitPlan just flipped the state machine into the
 // apply phase (Active→false, Applying→true). END the plan turn NOW — the chained
 // apply turn (runTurn's Applying branch) is the SINGLE sanctioned execution, run
Expand DownExpand Up@@ -1037,6 +1043,12 @@ func (s *Session) draftPlan(ctx context.Context, sys promptSpec, messages *[]wir
 results := s.executeBatchHooked(ctx, uses)
 *messages = append(*messages, wire.Message{Role: "user", Blocks: results})
 s.recordTurn(resp.Text(), uses)

 // A delegated worker hit a terminal failure — it cannot succeed on a retry,
 // so end the turn with the real cause instead of looping on it.
 if s.turn.fatalErr != nil {
 return resp, "", s.turn.fatalErr
 }
	}
	return resp, strings.TrimSpace(resp.Text()), nil
}
Expand Down

9 changes: 7 additions & 2 deletions
internal/agent/runtime/turnstate.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,13 @@ type turnState struct {
	redirected bool // the user denied an action and typed a redirection — skip the sibling tool calls but CONTINUE so the model reads the feedback and responds
	firstBreak string // the FIRST broken-edit nudge this turn — the failure evidence for lesson distillation
	lessonDone bool // a lesson was already distilled this turn (fire once)
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
	// fatalErr is a terminal failure raised from INSIDE a tool — a delegated
	// worker whose model call cannot succeed on any retry. It aborts the turn
	// after the batch, with the real cause, instead of being handed back as a
	// tool error the model will cheerfully retry forever.
	fatalErr error
	billingCredits bool // user consented to serve THIS turn on memcode credits after a BYOK key failure
	laneBypass string // "gateway" after a consented lane-exhaustion fallback — this turn serves off-lane
}

// newTurnState returns a fresh per-turn state (with an initialized gather tracker).
Expand Down

43 changes: 43 additions & 0 deletions
internal/llm/policy_test.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,13 @@ package llm
import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/memcode-ai/memcode/internal/provider"
	"github.com/memcode-ai/memcode/internal/wire"
	"google.golang.org/genai"
)

// scriptedProv is a ModelProvider+Streamer+Endpointer fake: records requested
Expand DownExpand Up@@ -325,3 +327,44 @@ func TestForkInheritsPin(t *testing.T) {
 t.Fatalf("forked turn requested %q, want the inherited pin (opus)", p.requested[0])
	}
}

// A 4xx that describes OUR request must never walk the fallback chain. The
// next model would receive the same malformed conversation and fail the same
// way — which is how one Gemini thought-signature 400 became 298 failed calls
// across two models before anything gave up.
//
// 408 and 429 are the deliberate exceptions: those are timing, not shape, and
// another model genuinely may serve them.
func TestRequestShapeErrorsAreTerminalButTimingErrorsWalk(t *testing.T) {
	apiErr := func(code int) error {
 return fmt.Errorf("gemini stream: %w", &genai.APIError{Code: code, Message: "boom"})
	}
	for _, tc := range []struct {
 name string
 code int
 wantCall int // total provider calls: 1 = terminal, 2 = walked
	}{
 {"400 malformed request", 400, 1},
 {"401 unauthorized", 401, 1},
 {"403 forbidden", 403, 1},
 {"404 unknown model", 404, 1},
 {"408 timeout", 408, 2},
 {"429 rate limited", 429, 2},
 {"500 provider down", 500, 2},
 {"503 unavailable", 503, 2},
	} {
 t.Run(tc.name, func(t *testing.T) {
 p := &scriptedProv{failures: map[string]error{"glm-5p2": apiErr(tc.code)}}
 r := pinnedRunner(p, prodInfo(nil), "glm-5p2")
 _, _ = r.Complete(context.Background(), MainLoop, userReq("hi"))
 if len(p.requested) != tc.wantCall {
 verb := "walked the chain"
 if tc.wantCall == 1 {
 verb = "stopped at the first model"
 }
 t.Fatalf("%d → %d calls (%v), want %d — it should have %s",
 tc.code, len(p.requested), p.requested, tc.wantCall, verb)
 }
 })
	}
}
Loading
Loading