From 95e62bcd0343cf6ed46cd3a865d11dd76fb6c7a8 Mon Sep 17 00:00:00 2001 From: corey Date: Fri, 28 Aug 2026 12:44:56 +0800 Subject: [PATCH 1/2] docs(derivation): stateException is only from verifyBatchRoots Stop documenting blob_count_mismatch as a BatchStatus divergence, and stop wrapping ErrBatchVerifyDivergence on that packing error so the sentinel matches the call site. Co-authored-by: Cursor --- node/derivation/metrics.go | 8 +++---- node/derivation/verify.go | 17 +++++--------- node/derivation/verify_local.go | 39 ++++++++++----------------------- 3 files changed, 20 insertions(+), 44 deletions(-) diff --git a/node/derivation/metrics.go b/node/derivation/metrics.go index 13d7c1559..6cd04be37 100644 --- a/node/derivation/metrics.go +++ b/node/derivation/metrics.go @@ -20,11 +20,9 @@ type Metrics struct { LatestBatchIndex metrics.Gauge SyncedBatchIndex metrics.Gauge - // LocalVerifyTriggered increments once per batch processed under - // VerifyModeLocal -- presence/absence on dashboards confirms the local - // verifier is running. Failure tracking is intentionally not split into - // separate counters; failures surface as Error logs and propagate as - // ErrBatchVerifyDivergence to BatchStatus=stateException. + // LocalVerifyTriggered increments once per local-verify batch. + // Failures are Error logs. BatchStatus=stateException is set only from + // verifyBatchRoots, not from this path. LocalVerifyTriggered metrics.Counter // Tag management metrics. SafeL2BlockNumber / FinalizedL2BlockNumber are diff --git a/node/derivation/verify.go b/node/derivation/verify.go index 2a73d6615..66f4cb337 100644 --- a/node/derivation/verify.go +++ b/node/derivation/verify.go @@ -10,18 +10,13 @@ import ( eth "github.com/morph-l2/go-ethereum/core/types" ) -// ErrBatchVerifyDivergence is wrapped by verification errors that represent -// a true "verifier reached a verdict of inconsistent" — i.e. the local -// chain disagrees with what L1 committed. Currently produced by: -// - verifyBatchRoots, when local stateRoot or withdrawalRoot ≠ L1 calldata -// - verify_local's rebuildBlob, for kinds versioned_hash_mismatch and -// blob_count_mismatch +// ErrBatchVerifyDivergence is wrapped by verifyBatchRoots when local +// stateRoot or withdrawalRoot disagrees with L1 calldata. derivation.go +// sets BatchStatus=stateException only on that path. // -// Call sites in derivation.go gate `metrics.SetBatchStatus(stateException)` -// on errors.Is(err, ErrBatchVerifyDivergence). Transient or runtime errors -// (RPC down, tx parsing failure, encoding bug, ...) intentionally do NOT -// wrap this sentinel: they reflect "verifier could not run", not "verifier -// determined divergence", and must not light up the divergence alert. +// Blob/content errors (rebuildBlob count or hash mismatch) are a different +// class and must not wrap this sentinel. Transient failures (RPC, parse, +// encode) also must not wrap it. var ErrBatchVerifyDivergence = errors.New("batch verify: divergence verdict") // verifyBatchRoots verifies the local state root and withdrawal root against the diff --git a/node/derivation/verify_local.go b/node/derivation/verify_local.go index 03f31c32d..973f2c685 100644 --- a/node/derivation/verify_local.go +++ b/node/derivation/verify_local.go @@ -23,18 +23,11 @@ import ( // declared by the L1 commitBatch tx (carried in BatchInfo.blobHashes). // // State / withdrawal root verification (verify.go::verifyBatchRoots) is -// independent of this path and runs after success. +// independent of this path and runs after success. Only that check can +// set BatchStatus=stateException. // -// On versioned_hash_mismatch the spec (SPEC-005 §4.3) calls for a -// single-batch self-heal: pull the real blob from beacon, decode + derive -// the batch via the layer1 engine API path (which would replace the -// locally divergent blocks via EL forkchoice), then re-run the shared -// verifyBatchRoots. That self-heal is **currently TODO** and not wired -// up here -- it is blocked on the EL number-continuity check (`params.Number -// == latestNumber + 1` in morph-reth `crates/engine-api/src/builder.rs` -// and go-ethereum `eth/catalyst/l2_api.go`) being relaxed in a separate -// spec. Until then a versioned_hash_mismatch falls through to the legacy -// failure path (log + return + retry next poll). +// Hash mismatch after rebuild is handled in derivation.go (self-heal via +// deriveForce). blob_count_mismatch is a packing error: log and retry. // // Mode is selected at startup via --derivation.verify-mode and is not // switchable at runtime. @@ -114,22 +107,12 @@ func (d *Derivation) fetchBatchInfoOutline(ctx context.Context, txHash common.Ha // range and compares them against batchInfo.blobHashes (taken from the L1 // commitBatch tx). Returns nil on match. // -// Failure paths intentionally inline metric inc + structured log + error -// construction at each kind site rather than route through a shared -// helper. One error-wrapping invariant the call site (derivation.go) -// relies on: +// Failure paths log a structured kind and return a plain error. The call +// site logs and retries; it does not set stateException. // -// - kind=versioned_hash_mismatch and kind=blob_count_mismatch wrap -// ErrBatchVerifyDivergence so the call site flips BatchStatus to -// stateException ONLY on a real "verifier reached unequal verdict"; -// transient / runtime errors must NOT light up the divergence alert. -// versioned_hash_mismatch will additionally be the self-heal trigger -// once the EL change lands (see file-level comment). -// -// All other kinds are plain errors. When you add a new kind, decide -// deliberately whether it represents "verifier could not run" (no -// sentinel) vs "verifier produced a divergence verdict" (wrap -// ErrBatchVerifyDivergence) and update the SentinelContract test. +// blob_count_mismatch means local packing disagrees with the L1 commit +// tx's blob count. That is not a state-root verdict and must not wrap +// ErrBatchVerifyDivergence. func (d *Derivation) rebuildBlob(ctx context.Context, batchInfo *BatchInfo) ([]common.Hash, error) { d.metrics.IncLocalVerifyTriggered() @@ -224,8 +207,8 @@ func (d *Derivation) rebuildBlob(ctx context.Context, batchInfo *BatchInfo) ([]c "rebuiltHashes", hashesHexCSV(rebuilt), "expectedHashes", hashesHexCSV(batchInfo.blobHashes), }, logBase...)...) - return nil, fmt.Errorf("local verify [blob_count_mismatch]: blob count mismatch (rebuilt=%d, l1=%d): %w", - len(rebuilt), len(batchInfo.blobHashes), ErrBatchVerifyDivergence) + return nil, fmt.Errorf("local verify [blob_count_mismatch]: blob count mismatch (rebuilt=%d, l1=%d)", + len(rebuilt), len(batchInfo.blobHashes)) } return rebuilt, nil } From 0d8b943b60cdf7ee964abe92932b6ed2fdc31434 Mon Sep 17 00:00:00 2001 From: corey Date: Fri, 28 Aug 2026 12:50:04 +0800 Subject: [PATCH 2/2] docs(derivation): narrow the fix to comments only Restore the ErrBatchVerifyDivergence wrap on blob_count_mismatch and the surrounding comment text; correct only the sentences that claimed the rebuildBlob path sets BatchStatus=stateException. Co-authored-by: Cursor --- node/derivation/metrics.go | 9 ++++++--- node/derivation/verify.go | 18 +++++++++++------ node/derivation/verify_local.go | 36 +++++++++++++++++++++++---------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/node/derivation/metrics.go b/node/derivation/metrics.go index 6cd04be37..b09ed3566 100644 --- a/node/derivation/metrics.go +++ b/node/derivation/metrics.go @@ -20,9 +20,12 @@ type Metrics struct { LatestBatchIndex metrics.Gauge SyncedBatchIndex metrics.Gauge - // LocalVerifyTriggered increments once per local-verify batch. - // Failures are Error logs. BatchStatus=stateException is set only from - // verifyBatchRoots, not from this path. + // LocalVerifyTriggered increments once per batch processed under + // VerifyModeLocal -- presence/absence on dashboards confirms the local + // verifier is running. Failure tracking is intentionally not split into + // separate counters; failures surface as Error logs only -- + // BatchStatus=stateException is set from verifyBatchRoots, not from + // this path. LocalVerifyTriggered metrics.Counter // Tag management metrics. SafeL2BlockNumber / FinalizedL2BlockNumber are diff --git a/node/derivation/verify.go b/node/derivation/verify.go index 66f4cb337..2c7daf47f 100644 --- a/node/derivation/verify.go +++ b/node/derivation/verify.go @@ -10,13 +10,19 @@ import ( eth "github.com/morph-l2/go-ethereum/core/types" ) -// ErrBatchVerifyDivergence is wrapped by verifyBatchRoots when local -// stateRoot or withdrawalRoot disagrees with L1 calldata. derivation.go -// sets BatchStatus=stateException only on that path. +// ErrBatchVerifyDivergence is wrapped by verification errors that represent +// a true "verifier reached a verdict of inconsistent" — i.e. the local +// chain disagrees with what L1 committed. Currently produced by: +// - verifyBatchRoots, when local stateRoot or withdrawalRoot ≠ L1 calldata +// - verify_local's rebuildBlob, for kind blob_count_mismatch // -// Blob/content errors (rebuildBlob count or hash mismatch) are a different -// class and must not wrap this sentinel. Transient failures (RPC, parse, -// encode) also must not wrap it. +// Only the verifyBatchRoots call site in derivation.go gates +// `metrics.SetBatchStatus(stateException)` on errors.Is(err, +// ErrBatchVerifyDivergence); the rebuildBlob failure path logs and retries +// without inspecting the sentinel. Transient or runtime errors +// (RPC down, tx parsing failure, encoding bug, ...) intentionally do NOT +// wrap this sentinel: they reflect "verifier could not run", not "verifier +// determined divergence", and must not light up the divergence alert. var ErrBatchVerifyDivergence = errors.New("batch verify: divergence verdict") // verifyBatchRoots verifies the local state root and withdrawal root against the diff --git a/node/derivation/verify_local.go b/node/derivation/verify_local.go index 973f2c685..540f68898 100644 --- a/node/derivation/verify_local.go +++ b/node/derivation/verify_local.go @@ -23,11 +23,18 @@ import ( // declared by the L1 commitBatch tx (carried in BatchInfo.blobHashes). // // State / withdrawal root verification (verify.go::verifyBatchRoots) is -// independent of this path and runs after success. Only that check can -// set BatchStatus=stateException. +// independent of this path and runs after success. // -// Hash mismatch after rebuild is handled in derivation.go (self-heal via -// deriveForce). blob_count_mismatch is a packing error: log and retry. +// On versioned_hash_mismatch the spec (SPEC-005 §4.3) calls for a +// single-batch self-heal: pull the real blob from beacon, decode + derive +// the batch via the layer1 engine API path (which would replace the +// locally divergent blocks via EL forkchoice), then re-run the shared +// verifyBatchRoots. That self-heal is **currently TODO** and not wired +// up here -- it is blocked on the EL number-continuity check (`params.Number +// == latestNumber + 1` in morph-reth `crates/engine-api/src/builder.rs` +// and go-ethereum `eth/catalyst/l2_api.go`) being relaxed in a separate +// spec. Until then a versioned_hash_mismatch falls through to the legacy +// failure path (log + return + retry next poll). // // Mode is selected at startup via --derivation.verify-mode and is not // switchable at runtime. @@ -107,12 +114,19 @@ func (d *Derivation) fetchBatchInfoOutline(ctx context.Context, txHash common.Ha // range and compares them against batchInfo.blobHashes (taken from the L1 // commitBatch tx). Returns nil on match. // -// Failure paths log a structured kind and return a plain error. The call -// site logs and retries; it does not set stateException. +// Failure paths intentionally inline metric inc + structured log + error +// construction at each kind site rather than route through a shared +// helper. On error wrapping: // -// blob_count_mismatch means local packing disagrees with the L1 commit -// tx's blob count. That is not a state-root verdict and must not wrap -// ErrBatchVerifyDivergence. +// - kind=blob_count_mismatch wraps ErrBatchVerifyDivergence, but this +// function's call site in derivation.go only logs and retries -- it +// does not read the sentinel. BatchStatus=stateException is set only +// on verifyBatchRoots failures (state / withdrawal root vs L1). +// +// All other kinds are plain errors. When you add a new kind, decide +// deliberately whether it represents "verifier could not run" (no +// sentinel) vs "verifier produced a divergence verdict" (wrap +// ErrBatchVerifyDivergence). func (d *Derivation) rebuildBlob(ctx context.Context, batchInfo *BatchInfo) ([]common.Hash, error) { d.metrics.IncLocalVerifyTriggered() @@ -207,8 +221,8 @@ func (d *Derivation) rebuildBlob(ctx context.Context, batchInfo *BatchInfo) ([]c "rebuiltHashes", hashesHexCSV(rebuilt), "expectedHashes", hashesHexCSV(batchInfo.blobHashes), }, logBase...)...) - return nil, fmt.Errorf("local verify [blob_count_mismatch]: blob count mismatch (rebuilt=%d, l1=%d)", - len(rebuilt), len(batchInfo.blobHashes)) + return nil, fmt.Errorf("local verify [blob_count_mismatch]: blob count mismatch (rebuilt=%d, l1=%d): %w", + len(rebuilt), len(batchInfo.blobHashes), ErrBatchVerifyDivergence) } return rebuilt, nil }