diff --git a/Cargo.lock b/Cargo.lock index 7e44d6c16..b3ae9ab9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1802,6 +1802,7 @@ dependencies = [ "ixon", "ixvm-codegen", "lean-ffi", + "memmap2", "mimalloc", "multi-stark", "n0-error", diff --git a/Ix/Cli/DiffCmd.lean b/Ix/Cli/DiffCmd.lean new file mode 100644 index 000000000..6a65a551f --- /dev/null +++ b/Ix/Cli/DiffCmd.lean @@ -0,0 +1,213 @@ +/- + `ix diff `: structured diff of two serialized Ixon + environments. + + The diff itself is computed in Rust (`rs_diff_env_files` → + `ixon::diff`): both files are memory-mapped and lazily parsed + (constant windows stay zero-copy mmap slices; `ConstantMeta` is never + bulk-materialized) and compared on anonymous structure — names serve + as join/display keys, constants compare by content address with + per-field classification (type/value/lvls/…, `block.*` for projection + targets, `"encoding"` when only the representation moved). `--meta` + additionally compares named metadata (`ConstantMeta`/`original`) via + a streaming merge-join over both files' §5 named sections. + + Every changed row carries a root-vs-rippled verdict: one edited + constant re-addresses its whole reverse-dependency cone, so most + changed rows are *rippled* (fully explained by dependency + re-addressing) and only the *roots* are intrinsic edits. The default + display lists roots and summarizes the rippled count; `--verbose` + lists rippled rows too, and in `--meta` mode rippled rows carrying + metadata edits stay visible. + + Exit codes (GNU diff convention): 0 = no difference found in the + selected mode, 1 = differences found, 2 = error. +-/ +module +public import Cli +public import Ix.Address +public import Ix.Common +public import Ix.Ixon + +public section + +namespace Ix.Cli.DiffCmd + +private def pad (s : String) (w : Nat) : String := + s.pushn ' ' (w - s.length) + +private def shortAddr (verbose : Bool) (a : Address) : String := + if verbose then toString a else ((toString a).take 12).toString ++ "…" + +/-- Synthetic mutual-block names embed the block hash as their second + component (`Ix.<64-hex>.…`); a changed block churns one such pair + per block, so the default display groups them into a count. -/ +private def isSyntheticMuts (s : String) : Bool := + match s.splitOn "." with + | "Ix" :: h :: _ => + h.length == 64 && h.all fun c => c.isDigit || ('a' ≤ c && c ≤ 'f') + | _ => false + +private def printStats (path : String) (s : Ixon.EnvStats) : IO Unit := + IO.println + s!"[diff] {path}: {s.consts} consts, {s.named} named, {s.blobs} blobs, {s.comms} comms" + +/-- Print up to `cap` addresses (all when `verbose`), one per line. -/ +private def printAddrList + (linePrefix : String) (addrs : Array Address) (verbose : Bool) : + IO Unit := do + let cap := if verbose then addrs.size else min addrs.size 10 + for a in addrs[0:cap] do + IO.println s!"{linePrefix}{shortAddr verbose a}" + if addrs.size > cap then + IO.println s!"{linePrefix}… and {addrs.size - cap} more" + +private def brackets (labels : Array String) : String := + "[" ++ ", ".intercalate labels.toList ++ "]" + +private def printNamedSection + (d : Ixon.EnvDiff) (wantMeta verbose : Bool) : IO Unit := do + let keep (s : String) : Bool := verbose || !isSyntheticMuts s + let added := d.namedAdded.filter (keep ·.1) + let removed := d.namedRemoved.filter (keep ·.1) + let synAdded := d.namedAdded.size - added.size + let synRemoved := d.namedRemoved.size - removed.size + let roots := d.namedChanged.filter (!·.rippled) + let rippleCount := + if d.namedChanged.isEmpty then "" + else s!" ({roots.size} roots, {d.namedChanged.size - roots.size} rippled)" + let metaCount := + if wantMeta then s!", {d.namedMetaOnly.size} metadata-only" else "" + IO.println + s!"named: {d.namedAdded.size} added, {d.namedRemoved.size} removed, {d.namedChanged.size} changed{rippleCount}{metaCount}" + if synAdded + synRemoved > 0 then + IO.println + s!" (synthetic mutual-block names: {synAdded} added, {synRemoved} removed — --verbose lists)" + -- Changed rows shown by default: the roots, plus (under --meta) + -- rippled rows carrying metadata edits — `namedMetaOnly` only covers + -- same-addr rows, so hiding those would hide real metadata changes. + let shown := d.namedChanged.filter fun c => + verbose || !c.rippled || (wantMeta && !c.metaFields.isEmpty) + -- Column width over everything we are about to print. + let mut wMax := 0 + for (n, _) in added do wMax := max wMax n.length + for (n, _) in removed do wMax := max wMax n.length + for c in shown do wMax := max wMax c.name.length + for (n, _) in d.namedMetaOnly do wMax := max wMax n.length + let w := min wMax 40 + for (n, addr) in added do + IO.println s!" + {pad n w} {shortAddr verbose addr}" + for (n, addr) in removed do + IO.println s!" - {pad n w} {shortAddr verbose addr}" + for c in shown do + let kind := + if c.oldKind == c.newKind then c.oldKind + else s!"{c.oldKind}→{c.newKind}" + let ripTag := if c.rippled then " (rippled)" else "" + IO.println + s!" ~ {pad c.name w} {pad kind 9} {shortAddr verbose c.oldAddr} → {shortAddr verbose c.newAddr} {brackets c.fields}{ripTag}" + if wantMeta && !c.metaFields.isEmpty then + IO.println s!" meta: {brackets c.metaFields}" + let hidden := d.namedChanged.size - shown.size + if hidden > 0 then + IO.println + s!" ({hidden} rippled rows hidden — address changes fully explained by dependency re-addressing; --verbose lists)" + for (n, labels) in d.namedMetaOnly do + IO.println s!" m {pad n w} {brackets labels}" + if shown.any (·.fields.contains "encoding") then + IO.println + " (encoding = representation changed; no semantic field difference detected)" + +def runDiffCmd (p : Cli.Parsed) : IO UInt32 := do + let some oldArg := p.positionalArg? "old" + | p.printError "error: must specify "; return 2 + let some newArg := p.positionalArg? "new" + | p.printError "error: must specify "; return 2 + let oldPath := oldArg.as! String + let newPath := newArg.as! String + let wantMeta := p.hasFlag "meta" + if wantMeta && p.hasFlag "anon" then + IO.eprintln "error: --anon and --meta are mutually exclusive" + return 2 + let verbose := p.hasFlag "verbose" + -- Byte-equal fast path and the diff itself both run over mmapped + -- files — nothing is read into Lean ByteArrays. + let d ← try + if ← Ixon.rsIxeFilesEqual oldPath newPath then + IO.println "identical" + return (0 : UInt32) + Ixon.rsDiffEnvFiles oldPath newPath wantMeta + catch e => + IO.eprintln s!"error: {e.toString}" + return (2 : UInt32) + printStats oldPath d.statsA + printStats newPath d.statsB + if d.isEmpty then + if wantMeta then + IO.println "files differ in bytes but no semantic difference found" + else + IO.println + "files differ in bytes but no anonymous-structure difference found (try --meta)" + return 0 + if let some (oldMain, newMain) := d.mainChanged then + let fmt : Option Address → String + | none => "∅" + | some a => shortAddr verbose a + IO.println s!"main: {fmt oldMain} → {fmt newMain}" + unless d.assumptionsAdded.isEmpty && d.assumptionsRemoved.isEmpty do + IO.println + s!"assumptions: +{d.assumptionsAdded.size} −{d.assumptionsRemoved.size}" + printAddrList " + " d.assumptionsAdded verbose + printAddrList " - " d.assumptionsRemoved verbose + unless d.namedAdded.isEmpty && d.namedRemoved.isEmpty + && d.namedChanged.isEmpty && d.namedMetaOnly.isEmpty do + printNamedSection d wantMeta verbose + unless d.commsAdded.isEmpty && d.commsRemoved.isEmpty + && d.commsChanged.isEmpty do + IO.println + s!"comms: +{d.commsAdded.size} −{d.commsRemoved.size} ~{d.commsChanged.size}" + printAddrList " + " d.commsAdded verbose + printAddrList " - " d.commsRemoved verbose + printAddrList " ~ " d.commsChanged verbose + unless d.constsOnlyA.isEmpty && d.constsOnlyB.isEmpty do + let note := + if verbose then "" else " (mutual blocks/projections; --verbose lists)" + IO.println + s!"consts: {d.constsOnlyA.size} only in {oldPath}, {d.constsOnlyB.size} only in {newPath}{note}" + if verbose then + printAddrList " - " d.constsOnlyA verbose + printAddrList " + " d.constsOnlyB verbose + unless d.blobsOnlyA.isEmpty && d.blobsOnlyB.isEmpty do + IO.println + s!"blobs: {d.blobsOnlyA.size} only in {oldPath}, {d.blobsOnlyB.size} only in {newPath}" + if verbose then + printAddrList " - " d.blobsOnlyA verbose + printAddrList " + " d.blobsOnlyB verbose + unless d.hintsChanged.isEmpty do + IO.println s!"hints changed: {d.hintsChanged.size}" + let cap := if verbose then d.hintsChanged.size else min d.hintsChanged.size 10 + for (a, oldH, newH) in d.hintsChanged[0:cap] do + IO.println s!" {shortAddr verbose a} {oldH} → {newH}" + if d.hintsChanged.size > cap then + IO.println s!" … and {d.hintsChanged.size - cap} more" + IO.println s!"[diff] {oldPath} ≠ {newPath}" + return 1 + +end Ix.Cli.DiffCmd + +open Ix.Cli.DiffCmd in +def diffCmd : Cli.Cmd := `[Cli| + diff VIA runDiffCmd; + "Print a structured diff of two serialized Ixon environments (`.ixe`). By default only anonymous structure is compared: constants by content address (joined through names, with per-field change classification), consts/blobs sets, comms, main/assumptions, and reducibility hints. Changed names are root-caused: rows fully explained by dependency re-addressing are counted as `rippled` and hidden by default, so the listing shows the intrinsic edits (roots). Exit codes: 0 = no difference, 1 = differences found, 2 = error." + + FLAGS: + anon; "Compare only anonymous structure (the default; accepted for explicitness)." + «meta»; "Additionally compare named metadata (binder names, originals, kv-maps)." + verbose; "Print full addresses, uncapped lists, synthetic mutual-block names, and rippled changed rows." + + ARGS: + old : String; "Path to the first (old) serialized env (`.ixe`)." + new : String; "Path to the second (new) serialized env (`.ixe`)." +] + +end diff --git a/Ix/Cli/PackCmd.lean b/Ix/Cli/PackCmd.lean new file mode 100644 index 000000000..390a7fb87 --- /dev/null +++ b/Ix/Cli/PackCmd.lean @@ -0,0 +1,76 @@ +/- + `ix pack `: prune a serialized env to the self-contained + bundle pinning one named constant, and write it as a standalone `.ixe`. + + A bundle is an `.ixe` whose `main` points at a distinguished constant; + because a constant's address is a merkle root over its whole dependency + DAG, `main`'s 32 bytes alone pin the value — the bundle is the + data-availability artifact that ships the bytes. `--assume` declares + trust-boundary cut-points: reached cut-points are recorded in the + bundle's `assumptions` instead of being carried (thin bundles). + + The heavy lifting is `Env::prune_to_closure` (3-edge value closure of + `main`, display metadata carried to fixpoint) followed by + `Env::validate_closed` — the same check a receiver runs — so a written + bundle is closed by construction. + + Different from `ix shard extract`: extract produces a general sub-env + for the kernel-check pipeline (no `main`, no `assumptions`, anon-work + block closure); pack produces a verified bundle with a root and an + explicit trust boundary. +-/ +module +public import Cli +public import Ix.Ixon +public import Ix.Cli.ConstsFile + +public section + +namespace Ix.Cli.PackCmd + +def runPackCmd (p : Cli.Parsed) : IO UInt32 := do + let some pathArg := p.positionalArg? "path" + | p.printError "error: must specify to a source .ixe file" + return 1 + let envPath := pathArg.as! String + let some nameArg := p.positionalArg? "name" + | p.printError "error: must specify of the bundle root constant" + return 1 + let mainName := nameArg.as! String + let assume ← Ix.Cli.ConstsFile.gather p "assume" "assume-file" + let outPath : String := + match p.flag? "out" with + | some flag => flag.as! String + | none => s!"{mainName}.ixe" + let anon := p.hasFlag "anon" + let verbose := p.hasFlag "verbose" + try + Ixon.rsPackEnv envPath mainName assume outPath anon verbose + let mode := if anon then " [anon]" else "" + IO.println s!"[pack] wrote {outPath} (main {mainName}, \ + {assume.size} assumption cut(s) declared){mode}" + return (0 : UInt32) + catch e => + IO.eprintln s!"error: {e.toString}" + return (1 : UInt32) + +end Ix.Cli.PackCmd + +open Ix.Cli.PackCmd in +def packCmd : Cli.Cmd := `[Cli| + pack VIA runPackCmd; + "Prune a `.ixe` env to the self-contained bundle pinning one constant (sets `main`; validated closed)" + + FLAGS: + anon; "Pack only anonymous structure — no names or metadata (§4/§5 empty; §3 hints still carried). The minimal artifact a receiver needs to typecheck/evaluate the pinned value." + assume : String; "Comma-separated cut-point constants — displayed names or 64-hex addresses. Reached cut-points are recorded in the bundle's `assumptions` instead of carried (thin bundle)." + "assume-file" : String; "Additionally read cut-points from a file (one per line; `#` comments and blank lines ignored). Unions with --assume." + out : String; "Output `.ixe` path. Defaults to `.ixe` (e.g. `Nat.add.ixe`)." + verbose; "Print pack details (source stats, kept counts, bytes written) to stderr." + + ARGS: + path : String; "Path to the source `.ixe` (e.g. from `ix compile`)." + name : String; "Displayed name of the bundle root constant (e.g. `Nat.add`)." +] + +end diff --git a/Ix/CompileM.lean b/Ix/CompileM.lean index 701608a55..a39fe51fd 100644 --- a/Ix/CompileM.lean +++ b/Ix/CompileM.lean @@ -84,6 +84,10 @@ structure BlockState where blockBlobs : Std.HashMap Address ByteArray := {} /-- Name components collected during block compilation -/ blockNames : Std.HashMap Address Ix.Name := {} + /-- Reducibility hints per definition name compiled in this block. + Hints are not part of `ConstantMeta`; the driver resolves this + map into `Ixon.Env.anonHints` once addresses are final. -/ + defHints : Std.HashMap Name Lean.ReducibilityHints := {} /-- Arena-based expression metadata for the current constant -/ arena : Ixon.ExprMetaArena := {} deriving Inhabited @@ -310,6 +314,10 @@ def storeString (s : String) : CompileM Address := do modifyBlockState fun c => { c with blockBlobs := c.blockBlobs.insert addr bytes } pure addr +/-- Record a definition's reducibility hints (see `BlockState.defHints`). -/ +def recordDefHints (name : Name) (hints : Lean.ReducibilityHints) : CompileM Unit := + modifyBlockState fun c => { c with defHints := c.defHints.insert name hints } + /-- Compile a name: store all string components as blobs and track name components in blockNames for deduplication. This matches Rust's compile_name behavior. -/ @@ -889,7 +897,8 @@ def compileDefinition (d : DefinitionVal) : CompileM (Ixon.Definition × Ixon.Co typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs d.hints allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name d.hints + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile a theorem to Ixon.Definition with metadata. -/ @@ -919,7 +928,8 @@ def compileTheorem (d : TheoremVal) : CompileM (Ixon.Definition × Ixon.Constant typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs .opaque allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name .opaque + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile an opaque to Ixon.Definition with metadata. -/ @@ -949,7 +959,8 @@ def compileOpaque (d : OpaqueVal) : CompileM (Ixon.Definition × Ixon.ConstantMe typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs .opaque allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name .opaque + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile an axiom to Ixon.Axiom with metadata. -/ @@ -1155,7 +1166,8 @@ def compileDefinitionData (d : Def) : CompileM (Ixon.Definition × Ixon.Constant | .defn => d.hints | .thm => .opaque | .opaq => .opaque - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs hints allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.name hints + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile inductive data for an Ind structure (from Mutual.lean). @@ -1510,6 +1522,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool -- Initialize compilation state let mut compileEnv := CompileEnv.new env let mut blockNames : Std.HashMap Address Ix.Name := {} + let mut defHints : Std.HashMap Name Lean.ReducibilityHints := {} -- Build work queue data structures let totalBlocks := blocks.blocks.size @@ -1554,6 +1567,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool blobs := cache.blockBlobs.fold (fun m k v => m.insert k v) compileEnv.blobs } blockNames := cache.blockNames.fold (fun m k v => m.insert k v) blockNames + defHints := cache.defHints.fold (fun m k v => m.insert k v) defHints -- If there are projections, store them and map names to projection addresses if result.projections.isEmpty then @@ -1608,6 +1622,17 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool -- Merge name string blobs into the main blobs map let allBlobs := nameBlobs.fold (fun m k v => m.insert k v) compileEnv.blobs + -- Resolve per-name hints to each name's registered constant address + -- (the projection address for mutual-block members — exactly the + -- address the kernel looks hints up under). Alias collisions merge + -- order-independently, matching Rust `CompileState::finalize_hints`. + let anonHints := compileEnv.nameToNamed.fold (init := {}) fun m name named => + match defHints.get? name with + | some h => m.alter named.addr fun + | some h₀ => some (Ixon.mergeHints h₀ h) + | none => some h + | none => m + let ixonEnv : Ixon.Env := { consts := compileEnv.constants.fold (init := {}) fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c) @@ -1616,6 +1641,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool names := namesMap comms := {} addrToName := addrToNameMap + anonHints } return .ok (ixonEnv, compileEnv.totalBytes) @@ -1705,6 +1731,7 @@ structure WaveBlockResult where projections : Array (Name × Ixon.Constant × Address × Ixon.ConstantMeta) blobs : Std.HashMap Address ByteArray names : Std.HashMap Address Ix.Name + defHints : Std.HashMap Name Lean.ReducibilityHints totalBytes : Nat /-- Work item for a worker thread -/ @@ -1793,6 +1820,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) projections := projsNoBytes blobs := cache.blockBlobs names := cache.blockNames + defHints := cache.defHints totalBytes := projBytes } discard <| resultChan.send result @@ -1808,6 +1836,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) let mut constants : Std.HashMap Address Ixon.Constant := {} let mut blobs : Std.HashMap Address ByteArray := {} let mut blockNames : Std.HashMap Address Ix.Name := {} + let mut defHints : Std.HashMap Name Lean.ReducibilityHints := {} let mut totalBytes : Nat := 0 let mut remaining : Set Name := {} @@ -1864,9 +1893,10 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) for (name, proj, addr, constMeta) in result.projections do constants := constants.insert addr proj nameToNamed := nameToNamed.insert name { addr, constMeta } - -- Store blobs and names + -- Store blobs, names, and hints blobs := result.blobs.fold (fun m k v => m.insert k v) blobs blockNames := result.names.fold (fun m k v => m.insert k v) blockNames + defHints := result.defHints.fold (fun m k v => m.insert k v) defHints totalBytes := totalBytes + result.totalBytes compiled := compiled + 1 @@ -1902,6 +1932,15 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) if dbg then IO.println s!" [Lean Compile] Blobs: {blockBlobCount} from blocks, {nameBlobCount} from names, {overlapCount} overlap, {finalBlobCount} final" + -- Resolve per-name hints to registered constant addresses (see the + -- serial driver / Rust `CompileState::finalize_hints`). + let anonHints := nameToNamed.fold (init := {}) fun m name named => + match defHints.get? name with + | some h => m.alter named.addr fun + | some h₀ => some (Ixon.mergeHints h₀ h) + | none => some h + | none => m + let ixonEnv : Ixon.Env := { consts := constants.fold (init := {}) fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c) @@ -1910,6 +1949,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) names := namesMap comms := {} addrToName := addrToNameMap + anonHints } return .ok (ixonEnv, totalBytes) diff --git a/Ix/DecompileM.lean b/Ix/DecompileM.lean index 81ba6c133..e2f548a93 100644 --- a/Ix/DecompileM.lean +++ b/Ix/DecompileM.lean @@ -507,7 +507,7 @@ def getLvlAddrs : ConstantMeta → Array Address | .empty | .muts _ => #[] def getArenaAndTypeRoot : ConstantMeta → ExprMetaArena × UInt64 - | .defn _ _ _ _ _ arena typeRoot _ => (arena, typeRoot) + | .defn _ _ _ _ arena typeRoot _ => (arena, typeRoot) | .axio _ _ arena typeRoot => (arena, typeRoot) | .quot _ _ arena typeRoot => (arena, typeRoot) | .indc _ _ _ _ _ arena typeRoot => (arena, typeRoot) @@ -516,11 +516,11 @@ def getArenaAndTypeRoot : ConstantMeta → ExprMetaArena × UInt64 | .empty | .muts _ => ({}, 0) def getAllAddrs : ConstantMeta → Array Address - | .defn _ _ _ all .. => all | .indc _ _ _ all .. => all + | .defn _ _ all .. => all | .indc _ _ _ all .. => all | .recr _ _ _ all .. => all | _ => #[] def getCtxAddrs : ConstantMeta → Array Address - | .defn _ _ _ _ ctx .. => ctx | .indc _ _ _ _ ctx .. => ctx + | .defn _ _ _ ctx .. => ctx | .indc _ _ _ _ ctx .. => ctx | .recr _ _ _ _ ctx .. => ctx | _ => #[] /-- Resolve name from ConstantMeta. -/ @@ -571,9 +571,16 @@ def decompileDefinition (d : Ixon.Definition) (cnst : Constant) (cMeta : Constan let univParams ← decompileMetaLevels cMeta let allNames ← decompileMetaAll cMeta name let mutCtx ← decompileMetaCtx cMeta - let (hints, valueRoot) := match cMeta with - | .defn _ _ hints _ _ _ _ valueRoot => (hints, valueRoot) - | _ => (.opaque, (0 : UInt64)) + let valueRoot := match cMeta with + | .defn _ _ _ _ _ _ valueRoot => valueRoot + | _ => (0 : UInt64) + -- Hints live in `Env.anonHints`, keyed by the constant address the + -- name resolves to; absent entry → `.opaque`, matching the + -- compiler's treatment of theorems and opaques. + let ixonEnv := (← getEnv).ixonEnv + let hints := match ixonEnv.named.get? name with + | some named => (ixonEnv.anonHints.get? named.addr).getD .opaque + | none => .opaque let (arena, typeRoot) := getArenaAndTypeRoot cMeta withFreshBlock cnst mutCtx univParams arena do let typeExpr ← decompileExpr d.typ typeRoot diff --git a/Ix/IxVM/ClaimHarness.lean b/Ix/IxVM/ClaimHarness.lean index 55e29ce17..52cb70c4f 100644 --- a/Ix/IxVM/ClaimHarness.lean +++ b/Ix/IxVM/ClaimHarness.lean @@ -181,13 +181,10 @@ def addEntries (ixonEnv : Ixon.Env) (keep : Address → Bool) ioBuffer := ioBuffer.extend 5 key (rawBytes.data.map fun b => .ofNat b.toNat) -- Discriminator: this addr resolves to a blob. ioBuffer := ioBuffer.extend 4 key #[.ofNat 0] - for (_, named) in ixonEnv.named do - if !keep named.addr then continue - match named.constMeta with - | .defn _ _ hints _ _ _ _ _ => - let key : Array Aiur.G := named.addr.hash.data.map .ofUInt8 - ioBuffer := ioBuffer.extend 3 key #[hintToG hints] - | _ => pure () + for (addr, hints) in ixonEnv.anonHints do + if !keep addr then continue + let key : Array Aiur.G := addr.hash.data.map .ofUInt8 + ioBuffer := ioBuffer.extend 3 key #[hintToG hints] return ioBuffer -- ============================================================================ diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index 0bf494acf..1be0476ee 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -475,7 +475,7 @@ def ExprMetaArena.mdataItemCount (arena : ExprMetaArena) : Nat := that constant, plus root indices pointing into the arena. -/ inductive ConstantMeta where | empty - | defn (name : Address) (lvls : Array Address) (hints : Lean.ReducibilityHints) + | defn (name : Address) (lvls : Array Address) (all : Array Address) (ctx : Array Address) (arena : ExprMetaArena) (typeRoot : UInt64) (valueRoot : UInt64) | axio (name : Address) (lvls : Array Address) @@ -497,7 +497,7 @@ inductive ConstantMeta where /-- Count total arena nodes in this ConstantMeta. -/ def ConstantMeta.exprMetaCount : ConstantMeta → Nat | .empty => 0 - | .defn _ _ _ _ _ arena _ _ => arena.nodes.size + | .defn _ _ _ _ arena _ _ => arena.nodes.size | .axio _ _ arena _ => arena.nodes.size | .quot _ _ arena _ => arena.nodes.size | .indc _ _ _ _ _ arena _ => arena.nodes.size @@ -508,7 +508,7 @@ def ConstantMeta.exprMetaCount : ConstantMeta → Nat /-- Count total arena nodes and mdata items in this ConstantMeta. -/ def ConstantMeta.exprMetaStats : ConstantMeta → Nat × Nat | .empty => (0, 0) - | .defn _ _ _ _ _ arena _ _ => (arena.nodes.size, arena.mdataItemCount) + | .defn _ _ _ _ arena _ _ => (arena.nodes.size, arena.mdataItemCount) | .axio _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) | .quot _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) | .indc _ _ _ _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) @@ -522,7 +522,7 @@ def ConstantMeta.exprMetaByType : ConstantMeta → Nat × Nat × Nat × Nat × N | .empty => (0, 0, 0, 0, 0) | cm => let arena := match cm with - | .defn _ _ _ _ _ a _ _ => a + | .defn _ _ _ _ a _ _ => a | .axio _ _ a _ => a | .quot _ _ a _ => a | .indc _ _ _ _ _ a _ => a @@ -1181,6 +1181,22 @@ def getReducibilityHints : GetM Lean.ReducibilityHints := do | 2 => pure (.regular (← getTag0).size.toUInt32) | x => throw s!"invalid ReducibilityHints {x}" +/-- Order-independent merge for hint registration: alpha-equivalent + definitions share one constant address but may carry different + reducibility hints (e.g. one alias marked `@[reducible]`), and the + winner must not depend on registration order. Keeps the minimum + under `(tag, height)` with `opaque < abbrev < regular h` — + commutative, associative, idempotent. Mirrors Rust + `Env::register_hint`. -/ +def mergeHints (a b : Lean.ReducibilityHints) : Lean.ReducibilityHints := + let key : Lean.ReducibilityHints → Nat × Nat + | .opaque => (0, 0) + | .abbrev => (1, 0) + | .regular n => (2, n.toNat) + let (a₁, a₂) := key a + let (b₁, b₂) := key b + if b₁ < a₁ || (b₁ == a₁ && b₂ < a₂) then b else a + /-- Serialize DataValue with indexed addresses. OfString/OfNat/OfInt/OfSyntax use raw 32-byte addresses (blob addresses, not in name index). -/ def putDataValueIndexed (dv : DataValue) (idx : NameIndex) : PutM Unit := do @@ -1315,11 +1331,10 @@ def getExprMetaArenaIndexed (rev : NameReverseIndex) : GetM ExprMetaArena := do def putConstantMetaIndexed (cm : ConstantMeta) (idx : NameIndex) : PutM Unit := do match cm with | .empty => putU8 255 - | .defn name lvls hints all ctx arena typeRoot valueRoot => + | .defn name lvls all ctx arena typeRoot valueRoot => putU8 0 putIdx name idx putIdxVec lvls idx - putReducibilityHints hints putIdxVec all idx putIdxVec ctx idx putExprMetaArenaIndexed arena idx @@ -1389,13 +1404,12 @@ def getConstantMetaIndexed (rev : NameReverseIndex) : GetM ConstantMeta := do | 0 => let name ← getIdx rev let lvls ← getIdxVec rev - let hints ← getReducibilityHints let all ← getIdxVec rev let ctx ← getIdxVec rev let arena ← getExprMetaArenaIndexed rev let typeRoot := (← getTag0).size let valueRoot := (← getTag0).size - pure (.defn name lvls hints all ctx arena typeRoot valueRoot) + pure (.defn name lvls all ctx arena typeRoot valueRoot) | 1 => let name ← getIdx rev let lvls ← getIdxVec rev @@ -1520,6 +1534,21 @@ structure Env where comms : Std.HashMap Address Comm := {} /-- Reverse index: constant Address → Ix.Name -/ addrToName : Std.HashMap Address Ix.Name := {} + /-- Distinguished root constant for bundle envs; `none` for whole + environments. A pointer, not a proof: readers check + `main ∈ consts`, and consumers holding an externally-expected + address must compare against it. -/ + main : Option Address := none + /-- Explicit trust boundary for thin bundles: addresses (constants + or blobs) the receiver is expected to already have. Serialized + as a strictly ascending leaf list; `Ix.Merkle.merkleRootCanonical` + over it reproduces the root a `Claim.assumptions` commits to. -/ + assumptions : Std.HashSet Address := {} + /-- Reducibility hints, keyed by constant address — the single home + for hints (they do not appear in `ConstantMeta`). The compiler + populates this map; `putEnv` serializes it as the hints section + and `getEnv` reads it back. Mirrors Rust `Env.anon_hints`. -/ + anonHints : Std.HashMap Address Lean.ReducibilityHints := {} deriving Inhabited namespace Env @@ -1627,13 +1656,21 @@ structure RawNameEntry where deriving Repr, Inhabited, BEq /-- Raw FFI environment structure using arrays instead of HashMaps. - This is the array-based equivalent of `Env` for FFI compatibility. -/ + This is the array-based equivalent of `Env` for FFI compatibility. + Field order matters: the Rust mirror (`LeanIxonRawEnv`) addresses + constructor slots positionally (0-7). -/ structure RawEnv where consts : Array RawConst named : Array RawNamed blobs : Array RawBlob comms : Array RawComm names : Array RawNameEntry := #[] + /-- Bundle root (`Env.main`). -/ + main : Option Address := none + /-- Bundle trust boundary (`Env.assumptions`), sorted ascending. -/ + assumptions : Array Address := #[] + /-- Explicit reducibility hints (`Env.anonHints`), sorted by address. -/ + anonHints : Array (Address × Lean.ReducibilityHints) := #[] deriving Repr, Inhabited, BEq namespace RawEnv @@ -1691,7 +1728,10 @@ def toEnv (raw : RawEnv) : Env := Id.run do env := { env with blobs := env.blobs.insert addr bytes } for ⟨addr, comm⟩ in raw.comms do env := env.storeComm addr comm - return env + return { env with + main := raw.main + assumptions := raw.assumptions.foldl (·.insert ·) {} + anonHints := raw.anonHints.foldl (fun m (a, h) => m.insert a h) {} } end RawEnv @@ -1700,7 +1740,9 @@ end RawEnv namespace Env /-- Convert Env with HashMaps to RawEnv with Arrays for FFI. - Includes the full names table for round-trip fidelity. -/ + Includes the full names table for round-trip fidelity. The + set-shaped bundle fields are sorted so the transfer is + deterministic (matches Rust `ixon_env_to_decoded`). -/ def toRawEnv (env : Env) : RawEnv := { consts := env.consts.toArray.map fun (addr, lc) => { addr, const := lc.get?.getD default } @@ -1708,6 +1750,11 @@ def toRawEnv (env : Env) : RawEnv := { blobs := env.blobs.toArray.map fun (addr, bytes) => { addr, bytes } comms := env.comms.toArray.map fun (addr, comm) => { addr, comm } names := env.names.toArray.map fun (addr, name) => { addr, name } + main := env.main + assumptions := env.assumptions.toList.toArray.qsort + fun a b => (compare a b).isLT + anonHints := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT } /-- Tag4 flag for Env (0xE), variant 0. -/ @@ -1773,11 +1820,15 @@ partial def topologicalSortNames (names : Std.HashMap Address Ix.Name) : Array ( let visited := visited.insert addr let result := result.push (addr, name) (visited, result) - -- Start with anonymous already visited (it's implicit) + -- Include the anonymous name first so it gets index 0 in the name + -- index (arena nodes frequently reference it as a binder name). + -- Matches Rust `topological_sort_names`, which emits it explicitly — + -- required for byte-identical writer output across the mirrors. let initVisited : Std.HashSet Address := ({} : Std.HashSet Address).insert anonAddr + let initResult : Array (Address × Ix.Name) := #[(anonAddr, Ix.Name.mkAnon)] -- Sort names by address before iterating to ensure deterministic DFS order let sortedEntries := names.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT - let (_, result) := sortedEntries.foldl (init := (initVisited, #[])) fun (visited, result) (_, name) => + let (_, result) := sortedEntries.foldl (init := (initVisited, initResult)) fun (visited, result) (_, name) => visit name visited result result @@ -1795,6 +1846,19 @@ def putEnv (env : Env) : PutM Unit := do let root := (Ix.Merkle.merkleRootCanonical constAddrs).getD Ix.Merkle.zeroAddress Serialize.put root + -- Bundle header fields: main (Option, 0/1-tagged) + assumptions + -- (strictly ascending address list). Matches Rust `Env::put`. + match env.main with + | none => putU8 0 + | some addr => do + putU8 1 + Serialize.put addr + let assumptions := env.assumptions.toList.toArray.qsort + fun a b => (compare a b).isLT + putTag0 ⟨assumptions.size.toUInt64⟩ + for addr in assumptions do + Serialize.put addr + -- Section 1: Blobs (Address -> bytes) let blobs := env.blobs.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨blobs.size.toUInt64⟩ @@ -1820,7 +1884,18 @@ def putEnv (env : Env) : PutM Unit := do putTag0 ⟨bytes.size.toUInt64⟩ putBytes bytes - -- Section 3: Names (Address -> Name component) + -- Section 3: anon_hints — the canonical hint channel for the + -- anon/lazy readers, placed before the metadata sections so they can + -- stop right after it. Serialized straight from `env.anonHints`, the + -- single home for hints. Matches Rust `Env::put`. + let hintPairs := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT + putTag0 ⟨hintPairs.size.toUInt64⟩ + for (addr, hints) in hintPairs do + Serialize.put addr + putReducibilityHints hints + + -- Section 4: Names (Address -> Name component) -- Topologically sorted so parents come before children, with ties broken by address let sortedNames := topologicalSortNames env.names -- Build name index from sorted positions (matching Rust) @@ -1831,7 +1906,7 @@ def putEnv (env : Env) : PutM Unit := do Serialize.put addr putNameComponent name - -- Section 4: Named (name Address -> Named with metadata) + -- Section 5: Named (name Address -> Named with metadata) let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨named.size.toUInt64⟩ for (name, namedEntry) in named do @@ -1847,7 +1922,7 @@ def putEnv (env : Env) : PutM Unit := do Serialize.put origAddr putConstantMetaIndexed origMeta nameIdx - -- Section 5: Comms (Address -> Comm) + -- Section 6: Comms (Address -> Comm) let comms := env.comms.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨comms.size.toUInt64⟩ for (addr, comm) in comms do @@ -1872,28 +1947,68 @@ def getEnv : GetM Env := do -- the recomputed root. let storedRoot : Address ← Serialize.get - let mut env : Env := {} - - -- Section 1: Blobs + -- Bundle header fields: main (Option) + strictly ascending + -- assumptions list. A pre-bundle-format `.ixe` has the §1 blob + -- count here, so a bad tag most likely means a stale file. + let mainTag ← getU8 + let main : Option Address ← match mainTag with + | 0 => pure none + | 1 => some <$> Serialize.get + | x => throw s!"Env.get: invalid main tag {x} in bundle header — \ + possibly a pre-bundle-format .ixe; recompile it" + let numAssumptions := (← getTag0).size + let mut assumptionArr : Array Address := #[] + for _ in [:numAssumptions.toNat] do + let addr : Address ← Serialize.get + if let some prev := assumptionArr.back? then + if !(compare prev addr).isLT then + throw "Env.get: assumptions not strictly ascending" + assumptionArr := assumptionArr.push addr + + let mut env : Env := { + main + assumptions := assumptionArr.foldl (·.insert ·) {} + } + + -- Section 1: Blobs (hash-verified per entry: a swapped blob would + -- otherwise silently change a Nat/String literal's value — the + -- consts merkle root covers only constant addresses) let numBlobs := (← getTag0).size for _ in [:numBlobs.toNat] do let addr ← Serialize.get let len := (← getTag0).size let bytes ← getBytes len.toNat + if Address.blake3 bytes != addr then + throw s!"Env.get: blob bytes hash mismatch for {reprStr (toString addr)}" env := { env with blobs := env.blobs.insert addr bytes } - -- Section 2: Consts (length-prefixed; see putEnv for rationale) + -- Section 2: Consts (length-prefixed; see putEnv for rationale). + -- Per-entry integrity: bytes must hash to the stored address. let numConsts := (← getTag0).size for _ in [:numConsts.toNat] do let addr ← Serialize.get let len := (← getTag0).size let bytes ← getBytes len.toNat + if Address.blake3 bytes != addr then + throw s!"Env.get: const bytes hash mismatch for {reprStr (toString addr)}" match deConstant bytes with | .ok constant => env := env.storeConst addr constant | .error e => throw s!"Env.get: bad constant bytes for addr {reprStr (toString addr)}: {e}" - -- Section 3: Names (build lookup table AND reverse index) + -- `main` must reference a constant actually present in the file. + if let some m := main then + if !env.consts.contains m then + throw s!"Env.get: main {reprStr (toString m)} not present in consts" + + -- Section 3: anon_hints + let numHints := (← getTag0).size + for _ in [:numHints.toNat] do + let addr : Address ← Serialize.get + let hints ← getReducibilityHints + env := { env with anonHints := env.anonHints.insert addr hints } + + -- Section 4: Names (build lookup table AND reverse index) let numNames := (← getTag0).size let mut namesLookup : Std.HashMap Address Ix.Name := {} let mut nameRev : NameReverseIndex := #[] @@ -1906,7 +2021,7 @@ def getEnv : GetM Env := do namesLookup := namesLookup.insert addr name env := { env with names := env.names.insert addr name } - -- Section 4: Named (name Address -> Named with metadata) + -- Section 5: Named (name Address -> Named with metadata) let numNamed := (← getTag0).size for _ in [:numNamed.toNat] do let nameAddr ← Serialize.get @@ -1930,7 +2045,7 @@ def getEnv : GetM Env := do | none => throw s!"getEnv: named entry references unknown name address {reprStr (toString nameAddr)}" - -- Section 5: Comms + -- Section 6: Comms let numComms := (← getTag0).size for _ in [:numComms.toNat] do let addr ← Serialize.get (α := Address) @@ -1946,6 +2061,13 @@ def getEnv : GetM Env := do if computedRoot != storedRoot then throw "Env.get: merkle root mismatch" + -- Comms is the final section; trailing bytes are truncation damage + -- or concatenated garbage. (Mirrors Rust `Env::get`; the early-stop + -- readers cannot make this check by design.) + let st ← get + if st.idx != st.bytes.size then + throw s!"Env.get: {st.bytes.size - st.idx} trailing bytes after final section" + pure env end Env @@ -1956,8 +2078,9 @@ def serEnv (env : Env) : ByteArray := runPut (Env.putEnv env) /-- Deserialize an Env from bytes (full metadata, pure Lean). -/ def deEnv (bytes : ByteArray) : Except String Env := runGet Env.getEnv bytes -/-- Compute section sizes for debugging. Returns (blobs, consts, names, named, comms). -/ -def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do +/-- Compute section sizes for debugging. + Returns (blobs, consts, anonHints, names, named, comms). -/ +def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat × Nat := Id.run do -- Blobs section let blobsBytes := runPut do let blobs := env.blobs.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT @@ -1975,6 +2098,15 @@ def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do Serialize.put addr putBytes lc.rawBytes + -- anon_hints section (mirrors putEnv's derive-from-named rule) + let hintsBytes := runPut do + let hintPairs := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT + putTag0 ⟨hintPairs.size.toUInt64⟩ + for (addr, hints) in hintPairs do + Serialize.put addr + putReducibilityHints hints + -- Names section let namesBytes := runPut do let sortedNames := Env.topologicalSortNames env.names @@ -2009,7 +2141,8 @@ def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do Serialize.put addr putComm comm - (blobsBytes.size, constsBytes.size, namesBytes.size, namedBytes.size, commsBytes.size) + ( blobsBytes.size, constsBytes.size, hintsBytes.size, namesBytes.size, + namedBytes.size, commsBytes.size ) /-! ## Rust FFI Serialization -/ @@ -2047,46 +2180,41 @@ structure RawConstSlice where len : UInt64 deriving Inhabited -/-- A named entry reduced to `name → addr` plus an encoded reducibility hint - (`hintKind`: 0 = none, 1 = opaque, 2 = abbrev, 3 = regular `hintVal`). -/ +/-- A named entry reduced to `name → addr`. -/ structure RawNamedLite where name : Ix.Name addr : Address - hintKind : UInt64 - hintVal : UInt64 deriving Inhabited -/-- Metadata-light env returned by `rs_de_env_lazy`. -/ +/-- Metadata-light env returned by `rs_de_env_lazy`. Field order + matters: the Rust builder addresses constructor slots 0-5. -/ structure RawEnvLazy where consts : Array RawConstSlice named : Array RawNamedLite blobs : Array RawBlob + /-- Bundle root (`Env.main`) from the header. -/ + main : Option Address := none + /-- Bundle trust boundary (`Env.assumptions`), in header order. -/ + assumptions : Array Address := #[] + /-- Reducibility hints (`Env.anonHints`) from the hints section. -/ + anonHints : Array (Address × Lean.ReducibilityHints) := #[] deriving Inhabited -/-- Decode an encoded reducibility hint into the `ConstantMeta` the check path - expects: a stripped `.defn` carrying just the hint (so `addEntries` finds it - via `.defn _ _ hints …`), or `.empty` for non-`Defn` entries. -/ -def RawNamedLite.toConstMeta (n : RawNamedLite) : ConstantMeta := - let mkDefn (h : Lean.ReducibilityHints) : ConstantMeta := - .defn default #[] h #[] #[] {} 0 0 - match n.hintKind with - | 1 => mkDefn .opaque - | 2 => mkDefn .abbrev - | 3 => mkDefn (.regular n.hintVal.toUInt32) - | _ => .empty - /-- Reconstruct an `Env` from a `RawEnvLazy` and the original buffer `buf`. Constants become `LazyConstant.ofSlice buf offset len` — zero-copy windows that share `buf`. `buf` MUST be the exact buffer passed to `rs_de_env_lazy` (offsets are relative to it). -/ def RawEnvLazy.toEnv (raw : RawEnvLazy) (buf : ByteArray) : Env := Id.run do - let mut env : Env := {} + let mut env : Env := { main := raw.main + assumptions := raw.assumptions.foldl (·.insert ·) {} + anonHints := raw.anonHints.foldl + (fun m (a, h) => m.insert a h) {} } for ⟨addr, offset, len⟩ in raw.consts do env := { env with consts := env.consts.insert addr (LazyConstant.ofSlice buf offset.toNat len.toNat) } for n in raw.named do - env := env.registerName n.name { addr := n.addr, constMeta := n.toConstMeta } + env := env.registerName n.name { addr := n.addr, constMeta := .empty } for ⟨addr, bytes⟩ in raw.blobs do env := { env with blobs := env.blobs.insert addr bytes } return env @@ -2104,9 +2232,10 @@ opaque rsDeEnvLazyFFI : @& ByteArray → Except String RawEnvLazy def deEnvAnon (bytes : ByteArray) : Except String Env := return (← rsDeEnvLazyFFI bytes).toEnv bytes -/-- Anonymous-only deserialization: keep blobs + consts, parse-and-drop - names/named/comms. Returns a `RawEnv` whose `named`/`names`/`comms` - arrays are empty. -/ +/-- Anonymous-only deserialization: keep blobs + consts + anon_hints + and stop — the metadata sections (names/named/comms) are laid out + after the hints and never touched. Returns a `RawEnv` whose + `named`/`names`/`comms` arrays are empty. -/ @[extern "rs_de_env_anon"] opaque rsDeEnvAnonFFI : @& ByteArray → Except String RawEnv @@ -2116,6 +2245,132 @@ opaque rsDeEnvAnonFFI : @& ByteArray → Except String RawEnv def rsDeEnvAnon (bytes : ByteArray) : Except String Env := return (← rsDeEnvAnonFFI bytes).toEnv +/-! ### Env diff (`rs_diff_envs`) + +The diff is computed in Rust (`ixon::diff::diff_envs`): both inputs are +parsed with the full reader (so `Named.original` participates), the +report below is marshaled back with names pre-rendered (`Name::pretty`) +and pre-sorted, addresses raw. Field order in these structures matters: +the Rust builders address constructor slots positionally (see +`crates/ffi/src/lean_ixon/diff.rs` and the `LeanIxonEnvDiff` layout in +`crates/ffi/src/lean.rs`). -/ + +/-- Per-env entity counts for the diff header. -/ +structure EnvStats where + consts : UInt64 + named : UInt64 + blobs : UInt64 + comms : UInt64 + deriving Inhabited, BEq + +/-- One name present in both envs whose constant address changed. + `fields` is never empty: `"kind"` marks a variant change and + `"encoding"` an address change with no detected semantic field + difference (table reorder / sharing-decision churn). `metaFields` + is only populated in meta mode. `rippled` is the root-cause + verdict: true iff the address change is fully explained by + dependency re-addressing (re-classified under the old→new quotient + of all changed rows, every residual label is + `"encoding"`/`"block-siblings"`); `fields` stays the strict + classification. -/ +structure NamedDiff where + name : String + oldAddr : Address + newAddr : Address + oldKind : String + newKind : String + fields : Array String + metaFields : Array String + rippled : Bool + deriving Inhabited, BEq + +/-- Report produced by `rsDiffEnvs`. Set-difference lists are complete + (display layers cap as needed); name-keyed arrays are sorted by + pretty name, address arrays ascending. -/ +structure EnvDiff where + /-- `none` = unchanged, otherwise `(first env's main, second's)`. -/ + mainChanged : Option (Option Address × Option Address) + assumptionsAdded : Array Address + assumptionsRemoved : Array Address + namedAdded : Array (String × Address) + namedRemoved : Array (String × Address) + namedChanged : Array NamedDiff + /-- Same constant address, different metadata (meta mode only). -/ + namedMetaOnly : Array (String × Array String) + commsAdded : Array Address + commsRemoved : Array Address + commsChanged : Array Address + constsOnlyA : Array Address + constsOnlyB : Array Address + blobsOnlyA : Array Address + blobsOnlyB : Array Address + /-- Hint deltas for constants present in BOTH envs, rendered as + `"opaque" | "abbrev" | "regular(N)" | "none"`. -/ + hintsChanged : Array (Address × String × String) + statsA : EnvStats + statsB : EnvStats + deriving Inhabited, BEq + +/-- True when no difference was found (ignores `statsA`/`statsB`, + which are always populated). -/ +def EnvDiff.isEmpty (d : EnvDiff) : Bool := + d.mainChanged.isNone + && d.assumptionsAdded.isEmpty && d.assumptionsRemoved.isEmpty + && d.namedAdded.isEmpty && d.namedRemoved.isEmpty + && d.namedChanged.isEmpty && d.namedMetaOnly.isEmpty + && d.commsAdded.isEmpty && d.commsRemoved.isEmpty && d.commsChanged.isEmpty + && d.constsOnlyA.isEmpty && d.constsOnlyB.isEmpty + && d.blobsOnlyA.isEmpty && d.blobsOnlyB.isEmpty + && d.hintsChanged.isEmpty + +@[extern "rs_diff_envs"] +opaque rsDiffEnvsFFI : @& ByteArray → @& ByteArray → Bool → Except String EnvDiff + +/-- Diff two serialized envs in Rust. `compareMeta := false` (the + default) compares only anonymous structure — name→addr changes with + per-field classification, consts/blobs sets, comms, + `main`/`assumptions`, and reducibility hints; `compareMeta := true` + additionally compares `Named` metadata content (`namedMetaOnly` + + `NamedDiff.metaFields`). -/ +def rsDiffEnvs (a b : ByteArray) (compareMeta : Bool := false) : + Except String EnvDiff := + rsDiffEnvsFFI a b compareMeta + +@[extern "rs_diff_env_files"] +opaque rsDiffEnvFilesFFI : @& String → @& String → Bool → IO EnvDiff + +/-- File-path variant of `rsDiffEnvs`: memory-maps both `.ixe` files + (constant windows stay zero-copy mmap slices backed by the OS page + cache) and diffs via the lazy reader in both modes — `ConstantMeta` + is never bulk-materialized; meta mode streams both §5 named + sections in a lockstep merge-join. The leanest diff path for + multi-GB envs. Failures surface as `IO` errors. -/ +def rsDiffEnvFiles (a b : String) (compareMeta : Bool := false) : + IO EnvDiff := + rsDiffEnvFilesFFI a b compareMeta + +/-- Byte-equality of two files: metadata length fast path, then an + mmap memcmp — no heap reads. -/ +@[extern "rs_ixe_files_equal"] +opaque rsIxeFilesEqual : @& String → @& String → IO Bool + +/-! ### Env pack (`rs_pack_env`) -/ + +/-- Pack a value bundle in Rust: memory-map the env at `envPath` + (lazy reader — metadata is never bulk-materialized), resolve + `mainName` (displayed form) to its constant address, prune to the + self-contained closure — `main` set, reached cut-points recorded in + `assumptions`; display metadata carried to fixpoint by re-streaming + §5 per round, or skipped entirely when `anon` is true (value + closure + §3 hints only, the minimal typecheck/eval artifact) — + validate (`Env::validate_closed`), and write the bundle to + `outPath`. `assume` entries resolve as displayed names first, else + as 64-hex constant addresses. Failures surface as `IO` errors. + Arg order: envPath, mainName, assume, outPath, anon, verbose. -/ +@[extern "rs_pack_env"] +opaque rsPackEnv : @& String → @& String → @& Array String → @& String → + Bool → Bool → IO Unit + /-! ## Canonical merkle root over consts -/ @[extern "rs_env_merkle_root"] diff --git a/Main.lean b/Main.lean index 7bdfd5b88..c3bdbd4b5 100644 --- a/Main.lean +++ b/Main.lean @@ -6,7 +6,9 @@ import Ix.Cli.CodegenCmd import Ix.Cli.CheckRsCmd import Ix.Cli.ClaimCmd import Ix.Cli.CompileCmd +import Ix.Cli.DiffCmd import Ix.Cli.IngressCmd +import Ix.Cli.PackCmd import Ix.Cli.ProfileCmd import Ix.Cli.ProveCmd import Ix.Cli.ShardCmd @@ -31,6 +33,8 @@ def ixCmd : Cli.Cmd := `[Cli| checkCmd; checkRsCmd; claimCmd; + diffCmd; + packCmd; treeCmd; profileCmd; proveCmd; diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index 63c988cd1..6f660965a 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -175,8 +175,8 @@ def constantMetaTests : TestSeq := let smallArena : ExprMetaArena := { nodes := #[.leaf, .app 0 0, .ref testAddr] } checkIO "ConstantMeta.empty" (roundtripIxonConstantMeta .empty == .empty) ++ checkIO "ConstantMeta.defn" (roundtripIxonConstantMeta - (.defn testAddr #[testAddr] .opaque #[] #[] smallArena 0 1) == - .defn testAddr #[testAddr] .opaque #[] #[] smallArena 0 1) ++ + (.defn testAddr #[testAddr] #[] #[] smallArena 0 1) == + .defn testAddr #[testAddr] #[] #[] smallArena 0 1) ++ checkIO "ConstantMeta.axio" (roundtripIxonConstantMeta (.axio testAddr #[] emptyArena 0) == .axio testAddr #[] emptyArena 0) ++ @@ -266,6 +266,277 @@ def rawEnvTests : TestSeq := test "RawEnv empty" (rawEnvEq (roundtripRawEnv empty) empty) ++ test "RawEnv with data" (rawEnvEq (roundtripRawEnv withData) withData) +/-! ## Env diff FFI (`rs_diff_envs`) + +The classification logic is covered by Rust unit tests +(`crates/ixon/src/diff.rs`); these tests pin the FFI marshaling — a +constructor-slot swap would surface as wrong-category contents. -/ + +def envDiffTests : TestSeq := + let fooName := Ix.Name.mkStr Ix.Name.mkAnon "foo" + let barName := Ix.Name.mkStr Ix.Name.mkAnon "bar" + -- `.var` exprs only: classification resolves table indices, so a + -- `.sort 0` over an empty univs table would be a (correctly) rejected + -- malformed constant. + let mkConst (value : Expr) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value } + sharing := #[], refs := #[], univs := #[] } + -- Hints live in `env.anonHints`, keyed by constant address; the + -- writer serializes that map as the hints section. + let mkEnv (c : Constant) (h : Lean.ReducibilityHints) : Env := Id.run do + let addr := Address.blake3 (serConstant c) + let mut env : Env := {} + env := env.storeConst addr c + env := { env with names := RawEnv.addNameComponents env.names fooName } + env := env.registerName fooName + { addr, constMeta := .defn fooName.getHash #[] #[] #[] {} 0 0 } + env := { env with anonHints := env.anonHints.insert addr h } + return env + let constA := mkConst (.var 0) + let constB := mkConst (.var 1) + let addrA := Address.blake3 (serConstant constA) + let addrB := Address.blake3 (serConstant constB) + let envBase := mkEnv constA (.regular 5) + let envValueChanged := mkEnv constB (.regular 5) + let envHintChanged := mkEnv constA (.regular 6) + let envExtra := Id.run do + let mut env := envBase + env := env.storeConst addrB constB + env := { env with names := RawEnv.addNameComponents env.names barName } + env := env.registerName barName { addr := addrB, constMeta := .empty } + return env + let runDiff (a b : Env) (compareMeta : Bool := false) : + Option Ixon.EnvDiff := + (Ixon.rsDiffEnvs (serEnv a) (serEnv b) compareMeta).toOption + test "EnvDiff: self-diff empty (anon)" + ((runDiff envBase envBase).any (·.isEmpty)) ++ + test "EnvDiff: self-diff empty (meta)" + ((runDiff envBase envBase true).any (·.isEmpty)) ++ + test "EnvDiff: stats populated" + ((runDiff envBase envBase).any fun d => + d.statsA.consts == 1 && d.statsA.named == 1 && d.statsB == d.statsA) ++ + test "EnvDiff: added name" + ((runDiff envBase envExtra).any fun d => + d.namedAdded == #[("bar", addrB)] && d.constsOnlyB == #[addrB] + && d.namedRemoved.isEmpty && d.namedChanged.isEmpty) ++ + test "EnvDiff: removed name" + ((runDiff envExtra envBase).any fun d => + d.namedRemoved == #[("bar", addrB)] && d.constsOnlyA == #[addrB]) ++ + test "EnvDiff: value change classified" + ((runDiff envBase envValueChanged).any fun d => + d.namedChanged.size == 1 && + (d.namedChanged[0]!).name == "foo" && + (d.namedChanged[0]!).oldAddr == addrA && + (d.namedChanged[0]!).newAddr == addrB && + (d.namedChanged[0]!).oldKind == "defn" && + (d.namedChanged[0]!).newKind == "defn" && + (d.namedChanged[0]!).fields == #["value"] && + (d.namedChanged[0]!).metaFields.isEmpty && + (d.namedChanged[0]!).rippled == false) ++ + -- A dependent whose only change is its ref re-addressing must marshal + -- `rippled == true` (pins the num_8 scalar slot against layout swaps). + test "EnvDiff: ripple verdict marshals" + (let mkRefConst (r : Address) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .ref 0 #[] } + sharing := #[], refs := #[r], univs := #[] } + let mkPair (leaf : Constant) : Env := Id.run do + let leafAddr := Address.blake3 (serConstant leaf) + let dep := mkRefConst leafAddr + let depAddr := Address.blake3 (serConstant dep) + let mut env : Env := {} + env := env.storeConst leafAddr leaf + env := env.storeConst depAddr dep + env := { env with names := RawEnv.addNameComponents env.names fooName } + env := env.registerName fooName { addr := leafAddr, constMeta := .empty } + env := { env with names := RawEnv.addNameComponents env.names barName } + env := env.registerName barName { addr := depAddr, constMeta := .empty } + return env + (runDiff (mkPair constA) (mkPair constB)).any fun d => + d.namedChanged.size == 2 && + (d.namedChanged.find? (·.name == "foo")).any (·.rippled == false) && + (d.namedChanged.find? (·.name == "bar")).any (·.rippled == true)) ++ + -- Hints live only in the env-level hints section, so a hint tweak + -- is a pure `hintsChanged` row — no metadata difference in either + -- mode (the named entries are identical). + test "EnvDiff: hint change visible in anon mode" + ((runDiff envBase envHintChanged).any fun d => + d.hintsChanged == #[(addrA, "regular(5)", "regular(6)")] + && d.namedMetaOnly.isEmpty && d.namedChanged.isEmpty) ++ + test "EnvDiff: hint change is not a metadata change" + ((runDiff envBase envHintChanged true).any fun d => + d.hintsChanged.size == 1 && d.namedMetaOnly.isEmpty + && d.namedChanged.isEmpty) ++ + test "EnvDiff: main change" + ((runDiff { envBase with main := some addrA } envBase).any fun d => + d.mainChanged == some (some addrA, none)) ++ + test "EnvDiff: assumptions delta" + (let p := Address.blake3 (ByteArray.mk #[1]) + let q := Address.blake3 (ByteArray.mk #[2]) + let r := Address.blake3 (ByteArray.mk #[3]) + let a := { envBase with + assumptions := ({} : Std.HashSet Address).insert p |>.insert q } + let b := { envBase with + assumptions := ({} : Std.HashSet Address).insert q |>.insert r } + (runDiff a b).any fun d => + d.assumptionsAdded == #[r] && d.assumptionsRemoved == #[p]) ++ + -- The mmap file-path entry must agree with the bytes-based one, and + -- the byte-equality fast path must distinguish same vs different. + (TestSeq.individualIO "EnvDiff: file-based diff matches bytes-based" + none (do + let dir ← IO.FS.createTempDir + let pa := dir / "a.ixe" + let pb := dir / "b.ixe" + let bytesA := serEnv envBase + let bytesB := serEnv envValueChanged + IO.FS.writeBinFile pa bytesA + IO.FS.writeBinFile pb bytesB + let r ← try + let eqSame ← Ixon.rsIxeFilesEqual pa.toString pa.toString + let eqDiff ← Ixon.rsIxeFilesEqual pa.toString pb.toString + let dFiles ← Ixon.rsDiffEnvFiles pa.toString pb.toString false + let ok := eqSame && !eqDiff && + (match Ixon.rsDiffEnvs bytesA bytesB false with + | .ok dBytes => dFiles == dBytes + | .error _ => false) + pure (ok, if ok then none else some "file/bytes reports disagree") + catch e => pure (false, some e.toString) + IO.FS.removeDirAll dir + pure (r.1, 0, 0, r.2)) .done) + +/-- Self-diff over the pure-Lean writer's bytes must be empty in both + modes (also exercises report marshaling on arbitrary inputs). -/ +def selfDiffEmpty (compareMeta : Bool) (env : RawEnv) : Bool := + match Ixon.rsDiffEnvs (serEnv env.toEnv) (serEnv env.toEnv) compareMeta with + | .ok d => d.isEmpty + | .error _ => false + +/-! ## Env pack FFI (`rs_pack_env`) + +The prune/validate engine is covered by Rust unit tests (the +`prune_to_closure` tests in `crates/ixon/src/env.rs`); these pin the +FFI surface end-to-end — file IO, name/hex resolution, bundle-field +population — and the closed-subset relationship to the source env. -/ + +/-- Write `env` to a temp `.ixe`, pack `mainName` with `assume` cuts, + and parse the resulting bundle. Pack failures surface as `.error`. -/ +private def packFixture (env : Env) (mainName : String) + (assume : Array String) (anon : Bool := false) : + IO (Except String Env) := do + let dir ← IO.FS.createTempDir + let src := dir / "src.ixe" + let out := dir / "bundle.ixe" + -- Every failure mode is caught into `.error`, so cleanup always runs. + let result ← try + IO.FS.writeBinFile src (serEnv env) + Ixon.rsPackEnv src.toString mainName assume out.toString anon false + let bytes ← IO.FS.readBinFile out + pure (Ixon.rsDeEnv bytes) + catch e => + pure (.error e.toString) + IO.FS.removeDirAll dir + return result + +/-- IO test asserting on a successfully packed bundle. -/ +private def packTest (descr : String) (act : IO (Except String Env)) + (check : Env → Bool) : TestSeq := + .individualIO descr none (do + match ← act with + | .ok bundle => + let ok := check bundle + pure (ok, 0, 0, if ok then none else some "bundle assertion failed") + | .error e => pure (false, 0, 0, some e)) .done + +/-- IO test asserting the pack call fails. -/ +private def packErrTest (descr : String) (act : IO (Except String Env)) : + TestSeq := + .individualIO descr none (do + match ← act with + | .ok _ => pure (false, 0, 0, some "expected pack to fail") + | .error _ => pure (true, 0, 0, none)) .done + +def envPackTests : TestSeq := + let fooName := Ix.Name.mkStr Ix.Name.mkAnon "foo" + let barName := Ix.Name.mkStr Ix.Name.mkAnon "bar" + let quxName := Ix.Name.mkStr Ix.Name.mkAnon "qux" + let leaf (v : UInt64) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .var v } + sharing := #[], refs := #[], univs := #[] } + let fooConst := leaf 0 + let quxConst := leaf 1 + let fooAddr := Address.blake3 (serConstant fooConst) + -- `bar` reaches `foo` through its refs table (a genuine closure + -- edge); `qux` is unreachable from `bar` and must be pruned away. + let barConst : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .ref 0 #[] } + sharing := #[], refs := #[fooAddr], univs := #[] } + let barAddr := Address.blake3 (serConstant barConst) + let src : Env := Id.run do + let mut env : Env := {} + for (n, c) in [(fooName, fooConst), (barName, barConst), + (quxName, quxConst)] do + let addr := Address.blake3 (serConstant c) + env := env.storeConst addr c + -- String components ride as blobs (the compiler's convention); + -- prune's `carry_name` recreates them in the bundle, so a fixture + -- without them would spuriously diff as bundle-only blobs. + let (names, blobs) := + RawEnv.addNameComponentsWithBlobs env.names env.blobs n + env := { env with names, blobs } + env := env.registerName n { addr, constMeta := .empty } + return env + packTest "EnvPack: closure bundle pins main and prunes" + (packFixture src "bar" #[]) (fun b => + b.main == some barAddr + && b.getAddr? barName == some barAddr + && b.getAddr? fooName == some fooAddr + && (b.getAddr? quxName).isNone + && b.consts.size == 2 + && b.assumptions.isEmpty) ++ + (TestSeq.individualIO "EnvPack: bundle is a removals-only subset under diff" + none (do + match ← packFixture src "bar" #[] with + | .error e => pure (false, 0, 0, some e) + | .ok b => + match Ixon.rsDiffEnvs (serEnv src) (serEnv b) with + | .error e => pure (false, 0, 0, some s!"diff failed: {e}") + | .ok d => + let ok := d.mainChanged == some (none, some barAddr) + && d.namedRemoved.size == 1 + && d.namedAdded.isEmpty && d.namedChanged.isEmpty + && d.constsOnlyB.isEmpty && d.blobsOnlyB.isEmpty + let msg := s!"mainOk={d.mainChanged == some (none, some barAddr)} \ + removed={d.namedRemoved.map (·.1)} \ + added={d.namedAdded.map (·.1)} \ + changed={d.namedChanged.map (·.name)} \ + constsOnlyB={d.constsOnlyB.size} blobsOnlyB={d.blobsOnlyB.size}" + pure (ok, 0, 0, if ok then none else some msg)) .done) ++ + packTest "EnvPack: assume cut (by name) records assumption" + (packFixture src "bar" #["foo"]) (fun b => + b.main == some barAddr + && (b.getAddr? fooName).isNone + && b.consts.size == 1 + && b.assumptions.size == 1 + && b.assumptions.contains fooAddr) ++ + packTest "EnvPack: assume cut (by hex address) records assumption" + (packFixture src "bar" #[toString fooAddr]) (fun b => + b.consts.size == 1 && b.assumptions.contains fooAddr) ++ + packTest "EnvPack: anon bundle has no metadata, keeps the value pin" + (packFixture src "bar" #[] (anon := true)) (fun b => + b.main == some barAddr + && b.named.isEmpty + -- The writer always emits the anonymous name as §4 entry 0. + && b.names.size <= 1 + && b.consts.size == 2 + && (b.getAddr? barName).isNone) ++ + packErrTest "EnvPack: unknown root name errors" + (packFixture src "nope" #[]) ++ + packErrTest "EnvPack: main cannot be assumed" + (packFixture src "bar" #["bar"]) + /-! ## Test Suite -/ def suite : List TestSeq := [ @@ -307,6 +578,14 @@ def suite : List TestSeq := [ checkIO "Ixon.Named roundtrip" (∀ x : Named, roundtripIxonNamed x == x), ---- RawEnv roundtrip checkIO "Ixon.RawEnv roundtrip" (∀ env : RawEnv, rawEnvEq (roundtripRawEnv env) env), + ---- Env diff + envDiffTests, + checkIO "Ixon.EnvDiff self-diff empty (anon)" + (∀ env : RawEnv, selfDiffEmpty false env), + checkIO "Ixon.EnvDiff self-diff empty (meta)" + (∀ env : RawEnv, selfDiffEmpty true env), + ---- Env pack + envPackTests, ] end Tests.FFI.Ixon diff --git a/Tests/FFI/Lifecycle.lean b/Tests/FFI/Lifecycle.lean index 790990383..d30554b78 100644 --- a/Tests/FFI/Lifecycle.lean +++ b/Tests/FFI/Lifecycle.lean @@ -137,10 +137,11 @@ def serdeTests : TestSeq := -- Empty RawEnv. Only data construction happens eagerly; FFI is deferred -- inside `mkSerdeRoundtripTest`. let empty : RawEnv := { consts := #[], named := #[], blobs := #[], comms := #[] } - -- RawEnv with data. The const's `addr` must be the canonical content - -- hash (`Address.blake3 (serConstant c)`) — the Rust loader verifies - -- `Address::hash(bytes) == addr` on load. Blobs and comms don't carry - -- a content-hash invariant, so `testAddr` is fine there. + -- RawEnv with data. Consts AND blobs must be content-addressed — the + -- Rust loader verifies `Address::hash(bytes) == addr` per entry for + -- both. `testAddr` is blake3 of `[1,2,3]`, which is exactly the blob's + -- bytes below, so the blob invariant holds; comms carry no + -- content-hash invariant, so `testAddr` is fine there too. let testAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) let testExpr : Expr := .sort 0 let testDef : Definition := { @@ -220,13 +221,12 @@ private def genSerdeRawEnv : Gen RawEnv := do let idx ← Gen.choose Nat 0 (pool.size - 1) let (addr, name) := pool[idx]! named := named.push { name, addr, constMeta := .empty } - -- Blobs: pool addresses + -- Blobs: content-addressed (the loaders verify blake3(bytes) == addr) let numBlobs ← Gen.choose Nat 0 3 let mut blobs : Array RawBlob := #[] for _ in [:numBlobs] do - let addr ← pickAddr let bytes ← Tests.Gen.Ixon.genByteArray - blobs := blobs.push { addr, bytes } + blobs := blobs.push { addr := Address.blake3 bytes, bytes } -- Comms: pool addresses let numComms ← Gen.choose Nat 0 2 let mut comms : Array RawComm := #[] diff --git a/Tests/Gen/Ixon.lean b/Tests/Gen/Ixon.lean index 4f7678449..f89ebc961 100644 --- a/Tests/Gen/Ixon.lean +++ b/Tests/Gen/Ixon.lean @@ -356,7 +356,7 @@ def genConstantMeta : Gen ConstantMeta := do frequency [ (10, pure .empty), (15, ConstantMeta.defn <$> genAddress <*> genSmallArray genAddress - <*> genReducibilityHints <*> genSmallArray genAddress <*> genSmallArray genAddress + <*> genSmallArray genAddress <*> genSmallArray genAddress <*> pure arena <*> genRoot <*> genRoot), (15, ConstantMeta.axio <$> genAddress <*> genSmallArray genAddress <*> pure arena <*> genRoot), @@ -457,9 +457,12 @@ def genRawConst : Gen RawConst := do def genRawNamed : Gen RawNamed := RawNamed.mk <$> genIxName 3 <*> genAddress <*> pure .empty -/-- Generate a RawBlob -/ -def genRawBlob : Gen RawBlob := - RawBlob.mk <$> genAddress <*> genByteArray +/-- Generate a RawBlob whose `addr` is the canonical content hash of + `bytes`. Readers verify `Address.blake3 bytes == addr` per entry + (blob-tampering defense), so arbitrary addresses are rejected. -/ +def genRawBlob : Gen RawBlob := do + let bytes ← genByteArray + pure { addr := Address.blake3 bytes, bytes } /-- Generate a RawComm -/ def genRawComm : Gen RawComm := @@ -469,13 +472,29 @@ def genRawComm : Gen RawComm := def genRawNameEntry : Gen RawNameEntry := RawNameEntry.mk <$> genAddress <*> genIxName 3 -/-- Generate a RawEnv with small arrays to avoid memory issues -/ -def genRawEnv : Gen RawEnv := - RawEnv.mk <$> genSmallArray genRawConst - <*> genSmallArray genRawNamed - <*> genSmallArray genRawBlob - <*> genSmallArray genRawComm - <*> genSmallArray genRawNameEntry +/-- Generate a RawEnv with small arrays to avoid memory issues. `main` + (when present) references a generated const — writers/readers check + `main ∈ consts`. Assumptions and hint keys are opaque addresses. -/ +def genRawEnv : Gen RawEnv := do + let consts ← genSmallArray genRawConst + let named ← genSmallArray genRawNamed + let blobs ← genSmallArray genRawBlob + let comms ← genSmallArray genRawComm + let names ← genSmallArray genRawNameEntry + let main ← + if consts.size > 0 then + frequency [ + (1, pure none), + (1, do + let i ← Gen.choose Nat 0 (consts.size - 1) + pure (consts[i % consts.size]?.map (·.addr))), + ] + else pure none + let assumptionList ← genList genAddress + let assumptions := assumptionList.toArray.qsort fun a b => (compare a b).isLT + let hintList ← genList (Prod.mk <$> genAddress <*> genReducibilityHints) + let anonHints := hintList.toArray.qsort fun a b => (compare a.1 b.1).isLT + pure { consts, named, blobs, comms, names, main, assumptions, anonHints } instance : Shrinkable RawConst where shrink rc := (fun c => { rc with const := c }) <$> Shrinkable.shrink rc.const @@ -486,18 +505,36 @@ instance : Shrinkable RawNamed where | _ => [{ rn with constMeta := .empty }] instance : Shrinkable RawBlob where - shrink rb := if rb.bytes.size > 0 then [{ rb with bytes := ByteArray.empty }] else [] + shrink rb := + -- Keep the content-address invariant: shrunk bytes need the + -- matching hash or the loaders reject the shrunk case. + if rb.bytes.size > 0 then + [{ addr := Address.blake3 ByteArray.empty, bytes := ByteArray.empty }] + else [] instance : Shrinkable RawComm where shrink _ := [] instance : Shrinkable RawEnv where shrink env := - (if env.consts.size > 0 then [{ env with consts := env.consts.pop }] else []) ++ + -- Popping a const may orphan `main` (writers reject main ∉ consts), + -- so clear it when its target is dropped. + (if env.consts.size > 0 then + let popped := env.consts.pop + let main := match env.main with + | some m => if popped.any (·.addr == m) then env.main else none + | none => none + [{ env with consts := popped, main }] + else []) ++ (if env.named.size > 0 then [{ env with named := env.named.pop }] else []) ++ (if env.blobs.size > 0 then [{ env with blobs := env.blobs.pop }] else []) ++ (if env.comms.size > 0 then [{ env with comms := env.comms.pop }] else []) ++ - (if env.names.size > 0 then [{ env with names := env.names.pop }] else []) + (if env.names.size > 0 then [{ env with names := env.names.pop }] else []) ++ + (if env.assumptions.size > 0 then [{ env with assumptions := env.assumptions.pop }] else []) ++ + (if env.anonHints.size > 0 then [{ env with anonHints := env.anonHints.pop }] else []) ++ + (match env.main with + | some _ => [{ env with main := none }] + | none => []) instance : SampleableExt RawConst := SampleableExt.mkSelfContained genRawConst instance : SampleableExt RawNamed := SampleableExt.mkSelfContained genRawNamed diff --git a/Tests/Ix/Compile.lean b/Tests/Ix/Compile.lean index f3d34e9cf..f32d232d1 100644 --- a/Tests/Ix/Compile.lean +++ b/Tests/Ix/Compile.lean @@ -222,7 +222,7 @@ def testCrossImpl : TestSeq := IO.println s!" [{i}] {reprStr arena.nodes[i]!}" let dumpMeta (label : String) (cm : Ixon.ConstantMeta) : IO Unit := do match cm with - | .defn _ _ _ _ _ arena typeRoot valueRoot => do + | .defn _ _ _ _ arena typeRoot valueRoot => do dumpArena label "arena" arena IO.println s!" {label} typeRoot={typeRoot} valueRoot={valueRoot}" | .axio _ _ arena typeRoot => do @@ -261,10 +261,9 @@ def testCrossImpl : TestSeq := IO.println s!" variant: {leanTag}" -- Field-by-field comparison for common variants match leanCM, rustCM with - | .defn ln ll lh la lc larena ltr lvr, .defn rn rl rh ra rc rarena rtr rvr => do + | .defn ln ll la lc larena ltr lvr, .defn rn rl ra rc rarena rtr rvr => do if ln != rn then IO.println s!" name DIFFERS: Lean={ln} Rust={rn}" if ll != rl then IO.println s!" lvls DIFFERS: Lean={ll.size} Rust={rl.size}" - if lh != rh then IO.println s!" hints DIFFERS" if la != ra then IO.println s!" all DIFFERS: Lean={la} Rust={ra}" if lc != rc then IO.println s!" ctx DIFFERS: Lean={lc} Rust={rc}" if larena != rarena then IO.println s!" arena DIFFERS: Lean={larena.nodes.size} Rust={rarena.nodes.size}" @@ -337,8 +336,8 @@ def testCrossImpl : TestSeq := IO.println s!"[Step 4] Lean blob stats: total={fmtBytes leanTotalBlobData}, max={fmtBytes leanMaxBlob}, avg={fmtBytes leanAvgBlob}, big(>1kB)={leanBig}, huge(>100kB)={leanHuge}" IO.println s!"[Step 4] Lean top 10 blob sizes: {leanTopSizes.map fmtBytes}" - let (leanBlobs, leanConsts, leanNames, leanNamed, leanComms) := Ixon.envSectionSizes leanIxonEnv - IO.println s!"[Step 4] Lean sections: blobs={fmtBytes leanBlobs}, consts={fmtBytes leanConsts}, names={fmtBytes leanNames}, named={fmtBytes leanNamed}, comms={fmtBytes leanComms}" + let (leanBlobs, leanConsts, leanHints, leanNames, leanNamed, leanComms) := Ixon.envSectionSizes leanIxonEnv + IO.println s!"[Step 4] Lean sections: blobs={fmtBytes leanBlobs}, consts={fmtBytes leanConsts}, hints={fmtBytes leanHints}, names={fmtBytes leanNames}, named={fmtBytes leanNamed}, comms={fmtBytes leanComms}" let leanEnvBytes := serializeEnv leanIxonEnv IO.println s!"[Step 4] Lean env done: {fmtBytes leanEnvBytes.size}" @@ -358,8 +357,8 @@ def testCrossImpl : TestSeq := IO.println s!"[Step 4] Rust blob stats: total={fmtBytes rustTotalBlobData}, max={fmtBytes rustMaxBlob}, avg={fmtBytes rustAvgBlob}, big(>1kB)={rustBig}, huge(>100kB)={rustHuge}" IO.println s!"[Step 4] Rust top 10 blob sizes: {rustTopSizes.map fmtBytes}" - let (rustBlobs, rustConsts, rustNames, rustNamed, rustComms) := Ixon.envSectionSizes phases.compileEnv - IO.println s!"[Step 4] Rust sections: blobs={fmtBytes rustBlobs}, consts={fmtBytes rustConsts}, names={fmtBytes rustNames}, named={fmtBytes rustNamed}, comms={fmtBytes rustComms}" + let (rustBlobs, rustConsts, rustHints, rustNames, rustNamed, rustComms) := Ixon.envSectionSizes phases.compileEnv + IO.println s!"[Step 4] Rust sections: blobs={fmtBytes rustBlobs}, consts={fmtBytes rustConsts}, hints={fmtBytes rustHints}, names={fmtBytes rustNames}, named={fmtBytes rustNamed}, comms={fmtBytes rustComms}" let rustEnvBytes := serializeEnv phases.compileEnv IO.println s!"[Step 4] Rust env done: {fmtBytes rustEnvBytes.size}" let serTime := (← IO.monoMsNow) - serStart diff --git a/Tests/Ix/Ixon.lean b/Tests/Ix/Ixon.lean index 72d22ec48..db7394bdd 100644 --- a/Tests/Ix/Ixon.lean +++ b/Tests/Ix/Ixon.lean @@ -153,9 +153,11 @@ def envSerdeUnit (env : Env) : Bool := def envUnitTests : TestSeq := -- Test 1: Empty env let emptyEnv : Env := {} - -- Test 2: Env with only a blob - let blobAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) - let envWithBlob : Env := { blobs := ({} : Std.HashMap _ _).insert blobAddr (ByteArray.mk #[4, 5, 6]) } + -- Test 2: Env with only a blob. Blob addresses must be the content + -- hash of the bytes — readers verify per entry. + let blobBytes := ByteArray.mk #[4, 5, 6] + let blobAddr := Address.blake3 blobBytes + let envWithBlob : Env := { blobs := ({} : Std.HashMap _ _).insert blobAddr blobBytes } -- Test 3: Env with a simple name (no named entry) let testName := Ix.Name.mkStr Ix.Name.mkAnon "test" let testNameAddr := testName.getHash @@ -179,15 +181,34 @@ def envUnitTests : TestSeq := let payloadAddr := Address.blake3 (ByteArray.mk #[13, 14, 15]) let commAddr := Address.blake3 (ByteArray.mk #[16, 17, 18]) let envWithBlobAndComm : Env := { - blobs := ({} : Std.HashMap _ _).insert blobAddr (ByteArray.mk #[4, 5, 6]), + blobs := ({} : Std.HashMap _ _).insert blobAddr blobBytes, comms := ({} : Std.HashMap _ _).insert commAddr (Comm.mk secretAddr payloadAddr) } + -- Test 7: Bundle env — main + assumptions + anonHints exercise the + -- bundle header and §3. + let bundleDef : Definition := { + kind := .defn, safety := .safe, lvls := 0, typ := .sort 0, value := .sort 0 + } + let bundleConst : Constant := { + info := .defn bundleDef, sharing := #[], refs := #[], univs := #[] + } + let bundleConstAddr := Address.blake3 (serConstant bundleConst) + let bundleEnv : Env := Id.run do + let mut env : Env := {} + env := env.storeConst bundleConstAddr bundleConst + return { env with + main := some bundleConstAddr + assumptions := ({} : Std.HashSet Address) + |>.insert (Address.blake3 (ByteArray.mk #[42])) + |>.insert (Address.blake3 (ByteArray.mk #[43])) + anonHints := ({} : Std.HashMap _ _).insert bundleConstAddr (.regular 5) } test "Empty env roundtrip" (envSerdeUnit emptyEnv) ++ test "Env with blob roundtrip" (envSerdeUnit envWithBlob) ++ test "Env with name roundtrip" (envSerdeUnit envWithName) ++ test "Env with named (empty meta) roundtrip" (envSerdeUnit envWithNamed) ++ test "Env with nested name roundtrip" (envSerdeUnit envWithNestedName) ++ - test "Env with blob+comm roundtrip" (envSerdeUnit envWithBlobAndComm) + test "Env with blob+comm roundtrip" (envSerdeUnit envWithBlobAndComm) ++ + test "Bundle env (main/assumptions/hints) roundtrip" (envSerdeUnit bundleEnv) /-! ## Cross-implementation serialization comparison tests -/ @@ -204,15 +225,22 @@ def envSerializationMatches (raw : RawEnv) : Bool := let env := raw.toEnv rsEqEnvSerialization raw (serEnv env) +/-- Strict byte equality between the pure-Lean writer and the Rust + writer over the same RawEnv (a stronger check than + `rsEqEnvSerialization`'s content comparison). -/ +def envBytesMatchRust (raw : RawEnv) : Bool := + serEnv raw.toEnv == rsSerEnvFFI raw + /-- Unit tests for Lean==Rust serialization comparison -/ def envSerializationUnitTests : TestSeq := -- Test 1: Empty env let emptyRaw : RawEnv := { consts := #[], named := #[], blobs := #[], comms := #[] } - -- Test 2: Env with one blob - let blobAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) + -- Test 2: Env with one blob (content-addressed: readers verify). + let blobBytes := ByteArray.mk #[4, 5, 6] + let blobAddr := Address.blake3 blobBytes let blobRaw : RawEnv := { consts := #[], named := #[], - blobs := #[{ addr := blobAddr, bytes := ByteArray.mk #[4, 5, 6] }], + blobs := #[{ addr := blobAddr, bytes := blobBytes }], comms := #[] } -- Test 3: Env with one comm @@ -227,13 +255,35 @@ def envSerializationUnitTests : TestSeq := -- Test 4: Env with blob + comm let blobCommRaw : RawEnv := { consts := #[], named := #[], - blobs := #[{ addr := blobAddr, bytes := ByteArray.mk #[4, 5, 6] }], + blobs := #[{ addr := blobAddr, bytes := blobBytes }], comms := #[{ addr := commAddr, comm := Comm.mk secretAddr payloadAddr }] } + -- Test 5: Bundle env — the directed cross-language vector for the + -- bundle header (main/assumptions) and §3 anon_hints. + let bundleDef : Definition := { + kind := .defn, safety := .safe, lvls := 0, typ := .sort 0, value := .sort 0 + } + let bundleConst : Constant := { + info := .defn bundleDef, sharing := #[], refs := #[], univs := #[] + } + let bundleConstAddr := Address.blake3 (serConstant bundleConst) + let bundleRaw : RawEnv := { + consts := #[{ addr := bundleConstAddr, const := bundleConst }], + named := #[], blobs := #[], comms := #[], + main := some bundleConstAddr, + assumptions := #[ + Address.blake3 (ByteArray.mk #[42]), + Address.blake3 (ByteArray.mk #[43])], + anonHints := #[(bundleConstAddr, .regular 5)] + } test "Empty env Lean==Rust" (envSerializationMatches emptyRaw) ++ test "Blob env Lean==Rust" (envSerializationMatches blobRaw) ++ test "Comm env Lean==Rust" (envSerializationMatches commRaw) ++ - test "Blob+Comm env Lean==Rust" (envSerializationMatches blobCommRaw) + test "Blob+Comm env Lean==Rust" (envSerializationMatches blobCommRaw) ++ + test "Bundle env Lean==Rust" (envSerializationMatches bundleRaw) ++ + test "Empty env bytes Lean==Rust" (envBytesMatchRust emptyRaw) ++ + test "Blob env bytes Lean==Rust" (envBytesMatchRust blobRaw) ++ + test "Bundle env bytes Lean==Rust" (envBytesMatchRust bundleRaw) /-! ## Canonical env merkle root: Lean vs. Rust agreement -/ diff --git a/crates/compile/src/compile.rs b/crates/compile/src/compile.rs index 63a4cc6b4..993314bde 100644 --- a/crates/compile/src/compile.rs +++ b/crates/compile/src/compile.rs @@ -20,8 +20,9 @@ use ix_common::env::{ AxiomVal, BinderInfo, ConstantInfo as LeanConstantInfo, ConstructorVal, DataValue as LeanDataValue, Env as LeanEnv, Expr as LeanExpr, ExprData, InductiveVal, Level, LevelData, Literal, Name, NameData, QuotVal, - RecursorRule as LeanRecursorRule, SourceInfo as LeanSourceInfo, - Substring as LeanSubstring, Syntax as LeanSyntax, SyntaxPreresolved, + RecursorRule as LeanRecursorRule, ReducibilityHints, + SourceInfo as LeanSourceInfo, Substring as LeanSubstring, + Syntax as LeanSyntax, SyntaxPreresolved, }; use ix_common::strong_ordering::SOrd; @@ -168,6 +169,13 @@ pub struct CompileState { /// right after `aux_gen::generate_aux_patches`. Blocks without nested /// auxiliaries simply aren't inserted. pub aux_perms: DashMap, + /// Reducibility hints per definition NAME, recorded by + /// `compile_definition` (the only place the Lean-side hints are in + /// scope — hints are not part of `ConstantMeta`). The constant + /// address a name resolves to isn't final until its `Named` entry is + /// registered, so [`Self::finalize_hints`] resolves this map through + /// `env.named` into `env.anon_hints` once compilation completes. + pub def_hints: DashMap, } /// Cached compiled expression with arena root index. @@ -228,6 +236,7 @@ impl Default for CompileState { brec_on_call_site_plans: Default::default(), below_call_site_plans: Default::default(), aux_perms: Default::default(), + def_hints: Default::default(), } } } @@ -264,6 +273,21 @@ impl CompileState { self.resolve_addr_aux(name, true) } + /// Resolve the per-name hints recorded by `compile_definition` into + /// `env.anon_hints`, keyed by each name's registered constant address + /// (`Named.addr` — the projection address for mutual-block members, + /// i.e. exactly the address the kernel looks hints up under). Runs + /// once after the scheduler drains: addresses aren't final until the + /// `Named` entries are registered. Alias collisions resolve through + /// `Env::register_hint`'s order-independent merge. + pub fn finalize_hints(&self) { + for entry in self.env.named.iter() { + if let Some(h) = self.def_hints.get(entry.key()) { + self.env.register_hint(entry.value().addr.clone(), *h.value()); + } + } + } + /// Promote a constant from `aux_name_to_addr` to `name_to_addr`, setting /// `Named.original` to the given `(orig_addr, orig_meta)` from the /// ephemeral no-aux compilation. The existing aux_gen `Named` entry keeps @@ -2219,7 +2243,6 @@ pub fn compile_definition( let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: name_addr, lvls: lvl_addrs, - hints: def.hints, all: all_addrs, ctx: ctx_addrs, arena, @@ -2227,6 +2250,7 @@ pub fn compile_definition( value_root, }); meta.meta_sharing = surgery_sharing; + stt.def_hints.insert(def.name.clone(), def.hints); Ok((data, meta)) } diff --git a/crates/compile/src/compile/env.rs b/crates/compile/src/compile/env.rs index 1afba1661..ca7b4d636 100644 --- a/crates/compile/src/compile/env.rs +++ b/crates/compile/src/compile/env.rs @@ -972,6 +972,8 @@ pub fn compile_env_with_options( }); } + stt.finalize_hints(); + if !*IX_QUIET { let total_elapsed = compile_start.elapsed().as_secs_f64(); eprintln!( diff --git a/crates/compile/src/decompile.rs b/crates/compile/src/decompile.rs index 536b13816..db94ac238 100644 --- a/crates/compile/src/decompile.rs +++ b/crates/compile/src/decompile.rs @@ -1351,14 +1351,25 @@ fn decompile_definition( dstt, )?; - let (hints, all) = match &meta.info { - ConstantMetaInfo::Def { hints, all, .. } => { + let all = match &meta.info { + ConstantMetaInfo::Def { all, .. } => { let all_names: Result, _> = all.iter().map(|a| decompile_name(a, stt)).collect(); - (*hints, all_names?) + all_names? }, - _ => (ReducibilityHints::Opaque, vec![]), + _ => vec![], }; + // Hints live in `env.anon_hints`, keyed by the constant address the + // name resolves to (for aux originals, the canonical address — the + // original was compiled from the same Lean definition, so the hints + // coincide). Absent entry → `Opaque`, matching the compiler's + // treatment of theorems and opaques. + let hints = stt + .env + .named + .get(&name) + .and_then(|n| stt.env.anon_hints.get(&n.value().addr).map(|r| *r)) + .unwrap_or(ReducibilityHints::Opaque); let cnst = ConstantVal { name, level_params, typ }; @@ -5473,7 +5484,6 @@ mod tests { let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: f_addr_name.clone(), lvls: vec![], - hints: ReducibilityHints::Opaque, all: vec![f_addr_name.clone()], ctx: vec![f_addr_name.clone()], arena, diff --git a/crates/ffi/Cargo.toml b/crates/ffi/Cargo.toml index 477add5af..1e02a9a6d 100644 --- a/crates/ffi/Cargo.toml +++ b/crates/ffi/Cargo.toml @@ -22,6 +22,7 @@ ix-compile = { workspace = true } ixon = { workspace = true } ix-kernel = { workspace = true } lean-ffi = { workspace = true } +memmap2 = { workspace = true } multi-stark = { workspace = true } mimalloc = { workspace = true } num-bigint = { workspace = true } diff --git a/crates/ffi/src/compile.rs b/crates/ffi/src/compile.rs index 23fb3f74b..f3a9234e5 100644 --- a/crates/ffi/src/compile.rs +++ b/crates/ffi/src/compile.rs @@ -443,6 +443,10 @@ pub extern "C" fn rs_compile_phases( raw_ixon_env.set_obj(2, blobs_arr); raw_ixon_env.set_obj(3, comms_arr); raw_ixon_env.set_obj(4, names_arr); + crate::lean_ixon::env::set_raw_env_bundle_fields( + &raw_ixon_env, + &compile_stt.env, + ); let result = LeanIxCompilePhases::alloc(0); result.set_obj(0, raw_env); @@ -539,6 +543,7 @@ pub extern "C" fn rs_compile_env_to_ixon( result.set_obj(2, blobs_arr); result.set_obj(3, comms_arr); result.set_obj(4, names_arr); + crate::lean_ixon::env::set_raw_env_bundle_fields(&result, &compile_stt.env); LeanIOResult::ok(result) } } diff --git a/crates/ffi/src/lean.rs b/crates/ffi/src/lean.rs index 0fc8d29e9..20ae247ab 100644 --- a/crates/ffi/src/lean.rs +++ b/crates/ffi/src/lean.rs @@ -52,13 +52,26 @@ lean_ffi::lean_inductive! { LeanIxonRawBlob [ { num_obj: 2 } ]; LeanIxonRawComm [ { num_obj: 2 } ]; LeanIxonRawNameEntry [ { num_obj: 2 } ]; - LeanIxonRawEnv [ { num_obj: 5 } ]; + // consts, named, blobs, comms, names, main, assumptions, anonHints + LeanIxonRawEnv [ { num_obj: 8 } ]; // Lazy/anon deserialization (`rs_de_env_lazy`): zero-copy const windows - // (addr + offset + len), name->addr + hint, and copied blobs. + // (addr + offset + len), name->addr, copied blobs, the bundle header + // fields (main, assumptions), and the hints-section pairs. LeanIxonRawConstSlice [ { num_obj: 1, num_64: 2 } ]; - LeanIxonRawNamedLite [ { num_obj: 2, num_64: 2 } ]; - LeanIxonRawEnvLazy [ { num_obj: 3 } ]; + LeanIxonRawNamedLite [ { num_obj: 2 } ]; + LeanIxonRawEnvLazy [ { num_obj: 6 } ]; + + // Env diff report (`rs_diff_envs`). Slot counts MUST match the Lean + // structures in Ix/Ixon.lean (Ixon.EnvStats / NamedDiff / EnvDiff). + // consts, named, blobs, comms counts + LeanIxonEnvStats [ { num_64: 4 } ]; + // name, oldAddr, newAddr, oldKind, newKind, fields, metaFields + // + rippled (Bool scalar) + LeanIxonNamedDiff [ { num_obj: 7, num_8: 1 } ]; + // mainChanged, assumptions±, named±/changed/metaOnly, comms±/changed, + // constsOnly×2, blobsOnly×2, hintsChanged, statsA, statsB + LeanIxonEnvDiff [ { num_obj: 17 } ]; // --- Ixon multi-variant inductives --- @@ -97,7 +110,7 @@ lean_ffi::lean_inductive! { LeanIxonConstantMeta [ { }, // tag 0: empty (scalar) - { num_obj: 6, num_64: 2 }, // tag 1: defn + { num_obj: 5, num_64: 2 }, // tag 1: defn { num_obj: 3, num_64: 1 }, // tag 2: axio { num_obj: 3, num_64: 1 }, // tag 3: quot { num_obj: 6, num_64: 1 }, // tag 4: indc diff --git a/crates/ffi/src/lean_ixon.rs b/crates/ffi/src/lean_ixon.rs index 51a89fd04..8cc7a26e6 100644 --- a/crates/ffi/src/lean_ixon.rs +++ b/crates/ffi/src/lean_ixon.rs @@ -6,10 +6,12 @@ #[cfg(feature = "test-ffi")] pub mod compare; pub mod constant; +pub mod diff; pub mod enums; pub mod env; pub mod expr; pub mod meta; +pub mod pack; #[cfg(feature = "test-ffi")] pub mod serialize; #[cfg(feature = "test-ffi")] diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs new file mode 100644 index 000000000..6bbbc2e64 --- /dev/null +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -0,0 +1,388 @@ +//! Env diff FFI: `rs_diff_envs` (two serialized byte arrays), +//! `rs_diff_env_files` (two file paths, memory-mapped), and +//! `rs_ixe_files_equal` (byte-equality fast path) — all marshaling the +//! [`EnvDiff`] report to Lean (`Ixon.EnvDiff`). +//! +//! Both modes load via the memory-lean lazy-index path +//! (`parse_lazy_index` + `from_lazy_index`/`from_lazy_index_mmap` — +//! `ConstantMeta` is never bulk-materialized); meta mode additionally +//! streams both files' §5 named sections in a lockstep merge-join +//! (`ixon::diff::diff_envs_lazy`). The file-path entry keeps constant +//! windows as zero-copy mmap slices, so neither the 2× file bytes nor +//! the const-window copies of the ByteArray path are resident. Large +//! inputs get incremental progress on stderr. +//! +//! Names cross the boundary pre-rendered (`Name::pretty()`) and +//! pre-sorted; addresses cross raw so Lean owns display formatting. + +use std::sync::Arc; +use std::time::Instant; + +use ix_common::address::Address; +use ixon::diff::{ + DiffPhase, EnvDiff, EnvStats, JoinProgress, LazySide, NamedChange, + diff_envs_lazy, +}; +use ixon::env::{Env as IxonEnv, LazyIndex}; +use lean_ffi::object::{ + LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanIOResult, + LeanOption, LeanOwned, LeanProd, LeanString, +}; + +use crate::lean::{ + LeanIxAddress, LeanIxonEnvDiff, LeanIxonEnvStats, LeanIxonNamedDiff, +}; + +fn addr_array(v: &[Address]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, a) in v.iter().enumerate() { + arr.set(i, LeanIxAddress::build(a)); + } + arr +} + +fn string_array(v: &[String]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, s) in v.iter().enumerate() { + arr.set(i, LeanString::new(s)); + } + arr +} + +/// `Array (String × Address)` +fn name_addr_array(v: &[(String, Address)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (s, a)) in v.iter().enumerate() { + arr.set(i, LeanProd::new(LeanString::new(s), LeanIxAddress::build(a))); + } + arr +} + +/// `Array (String × Array String)` +fn name_labels_array(v: &[(String, Vec)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (s, labels)) in v.iter().enumerate() { + arr.set(i, LeanProd::new(LeanString::new(s), string_array(labels))); + } + arr +} + +/// `Array (Address × String × String)` (right-nested prod) +fn hint_array(v: &[(Address, String, String)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (a, old, new)) in v.iter().enumerate() { + arr.set( + i, + LeanProd::new( + LeanIxAddress::build(a), + LeanProd::new(LeanString::new(old), LeanString::new(new)), + ), + ); + } + arr +} + +fn opt_addr(o: &Option
) -> LeanOwned { + match o { + None => LeanOption::none().into(), + Some(a) => LeanOption::some(LeanIxAddress::build(a)).into(), + } +} + +impl LeanIxonEnvStats { + /// Build `Ixon.EnvStats { consts, named, blobs, comms }`. + fn build(s: &EnvStats) -> Self { + let ctor = LeanIxonEnvStats::alloc(0); + ctor.set_num_64(0, s.consts as u64); + ctor.set_num_64(1, s.named as u64); + ctor.set_num_64(2, s.blobs as u64); + ctor.set_num_64(3, s.comms as u64); + ctor + } +} + +impl LeanIxonNamedDiff { + /// Build `Ixon.NamedDiff { name, oldAddr, newAddr, oldKind, newKind, + /// fields, metaFields, rippled }`. + fn build(c: &NamedChange) -> Self { + let ctor = LeanIxonNamedDiff::alloc(0); + ctor.set_obj(0, LeanString::new(&c.name)); + ctor.set_obj(1, LeanIxAddress::build(&c.old_addr)); + ctor.set_obj(2, LeanIxAddress::build(&c.new_addr)); + ctor.set_obj(3, LeanString::new(c.old_kind)); + ctor.set_obj(4, LeanString::new(c.new_kind)); + ctor.set_obj(5, string_array(&c.fields)); + ctor.set_obj(6, string_array(&c.meta_fields)); + ctor.set_num_8(0, u8::from(c.rippled)); + ctor + } +} + +impl LeanIxonEnvDiff { + /// Build `Ixon.EnvDiff` — field order MUST match the Lean structure. + fn build(d: &EnvDiff) -> Self { + let main_changed: LeanOwned = match &d.main_changed { + None => LeanOption::none().into(), + Some((a, b)) => { + LeanOption::some(LeanProd::new(opt_addr(a), opt_addr(b))).into() + }, + }; + let named_changed = LeanArray::alloc(d.named_changed.len()); + for (i, c) in d.named_changed.iter().enumerate() { + named_changed.set(i, LeanIxonNamedDiff::build(c)); + } + let ctor = LeanIxonEnvDiff::alloc(0); + ctor.set_obj(0, main_changed); + ctor.set_obj(1, addr_array(&d.assumptions_added)); + ctor.set_obj(2, addr_array(&d.assumptions_removed)); + ctor.set_obj(3, name_addr_array(&d.named_added)); + ctor.set_obj(4, name_addr_array(&d.named_removed)); + ctor.set_obj(5, named_changed); + ctor.set_obj(6, name_labels_array(&d.named_meta_only)); + ctor.set_obj(7, addr_array(&d.comms_added)); + ctor.set_obj(8, addr_array(&d.comms_removed)); + ctor.set_obj(9, addr_array(&d.comms_changed)); + ctor.set_obj(10, addr_array(&d.consts_only_a)); + ctor.set_obj(11, addr_array(&d.consts_only_b)); + ctor.set_obj(12, addr_array(&d.blobs_only_a)); + ctor.set_obj(13, addr_array(&d.blobs_only_b)); + ctor.set_obj(14, hint_array(&d.hints_changed)); + ctor.set_obj(15, LeanIxonEnvStats::build(&d.stats_a)); + ctor.set_obj(16, LeanIxonEnvStats::build(&d.stats_b)); + ctor + } +} + +/// Inputs at least this large get incremental progress on stderr +/// (`[rs_diff_envs] …` lines, matching the `[rs_compile_env]` idiom). +/// Small inputs — unit tests, property tests, tiny bundles — stay +/// silent. +const PROGRESS_MIN_BYTES: usize = 100 * 1024 * 1024; + +/// Stderr line per progress event, tagged by phase. +fn print_progress(p: &JoinProgress) { + match p.phase { + DiffPhase::MetaSweep => eprintln!( + "[rs_diff_envs] meta sweep: {}/{} ({} differing so far)", + p.done, p.total, p.changed + ), + DiffPhase::NamedJoin => eprintln!( + "[rs_diff_envs] named join: {}/{} ({} changed so far)", + p.done, p.total, p.changed + ), + DiffPhase::RippleClassify => eprintln!( + "[rs_diff_envs] ripple pass: {}/{} ({} roots so far)", + p.done, p.total, p.changed + ), + } +} + +fn print_parse_start(which: &str, len: usize, progress: bool) { + if progress { + eprintln!( + "[rs_diff_envs] parsing {which} env ({} MB, lazy reader)...", + len / 1_000_000 + ); + } +} + +fn print_parse_done(which: &str, t: Instant, env: &IxonEnv, progress: bool) { + if progress { + eprintln!( + "[rs_diff_envs] {which} env parsed in {:.1}s ({} consts, {} named, {} blobs)", + t.elapsed().as_secs_f64(), + env.consts.len(), + env.named.len(), + env.blobs.len() + ); + } +} + +/// Lazy-parse one heap-backed side: index + env. +fn parse_side_bytes( + bytes: &[u8], + which: &str, + progress: bool, +) -> Result<(LazyIndex, IxonEnv), String> { + print_parse_start(which, bytes.len(), progress); + let t = Instant::now(); + let index = IxonEnv::parse_lazy_index(bytes) + .map_err(|e| format!("{which} input: {e}"))?; + let env = IxonEnv::from_lazy_index(&index, bytes) + .map_err(|e| format!("{which} input: {e}"))?; + print_parse_done(which, t, &env, progress); + Ok((index, env)) +} + +/// Run the shared lazy diff over two prepared sides and marshal. +fn run_diff( + a: LazySide<'_>, + b: LazySide<'_>, + want_meta: bool, + progress: bool, + who: &str, +) -> Result { + let t = Instant::now(); + let mut on_progress = |p: JoinProgress| { + if progress { + print_progress(&p); + } + }; + let d = diff_envs_lazy(a, b, want_meta, &mut on_progress)?; + if progress { + eprintln!("[{who}] diff computed in {:.1}s", t.elapsed().as_secs_f64()); + } + Ok(d) +} + +/// FFI: diff two serialized envs. +/// `(a b : ByteArray) → (meta : Bool) → Except String Ixon.EnvDiff`. +/// Pure result; both modes use the lazy reader (meta mode adds the +/// streaming §5 sweep). For large inputs, progress goes to stderr. +#[unsafe(no_mangle)] +pub extern "C" fn rs_diff_envs( + a: LeanByteArray>, + b: LeanByteArray>, + meta: LeanBool>, +) -> LeanExcept { + let (a_bytes, b_bytes) = (a.as_bytes(), b.as_bytes()); + let progress = + a_bytes.len() >= PROGRESS_MIN_BYTES || b_bytes.len() >= PROGRESS_MIN_BYTES; + let want_meta = meta.to_bool(); + let sides = parse_side_bytes(a_bytes, "first", progress) + .and_then(|a| Ok((a, parse_side_bytes(b_bytes, "second", progress)?))); + let ((ia, ea), (ib, eb)) = match sides { + Ok(s) => s, + Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + }; + match run_diff( + LazySide { env: &ea, index: &ia, data: a_bytes }, + LazySide { env: &eb, index: &ib, data: b_bytes }, + want_meta, + progress, + "rs_diff_envs", + ) { + Ok(d) => LeanExcept::ok(LeanIxonEnvDiff::build(&d)), + Err(e) => LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + } +} + +/// Memory-map a file, guarding against a concurrent length change. +/// Shared with `rs_pack_env` (`super::pack`). +pub(crate) fn mmap_file( + path: &str, + which: &str, +) -> Result, String> { + let file = std::fs::File::open(path) + .map_err(|e| format!("{which} input {path}: {e}"))?; + let expected = + file.metadata().map_err(|e| format!("{which} input {path}: {e}"))?.len(); + let mmap = unsafe { memmap2::Mmap::map(&file) } + .map_err(|e| format!("{which} input {path}: mmap failed: {e}"))?; + if mmap.len() as u64 != expected { + return Err(format!( + "{which} input {path}: mapped length {} != metadata length {expected}", + mmap.len() + )); + } + Ok(Arc::new(mmap)) +} + +/// Lazy-parse one mmap-backed side: constant windows stay zero-copy. +fn parse_side_mmap( + mmap: &Arc, + which: &str, + progress: bool, +) -> Result<(LazyIndex, IxonEnv), String> { + print_parse_start(which, mmap.len(), progress); + let t = Instant::now(); + let index = IxonEnv::parse_lazy_index(&mmap[..]) + .map_err(|e| format!("{which} input: {e}"))?; + let env = IxonEnv::from_lazy_index_mmap(&index, mmap) + .map_err(|e| format!("{which} input: {e}"))?; + print_parse_done(which, t, &env, progress); + Ok((index, env)) +} + +/// FFI: diff two `.ixe` files by path, memory-mapped. +/// `(a b : String) → (meta : Bool) → IO Ixon.EnvDiff`. +/// Constant windows are zero-copy mmap slices, so neither the file +/// bytes nor const-window copies are heap-resident — the leanest diff +/// path for multi-GB envs. Errors surface as `IO` errors. +#[unsafe(no_mangle)] +pub extern "C" fn rs_diff_env_files( + a_path: LeanString>, + b_path: LeanString>, + meta: LeanBool>, +) -> LeanIOResult { + let want_meta = meta.to_bool(); + let a_path = a_path.to_string(); + let b_path = b_path.to_string(); + let sides = mmap_file(&a_path, "first") + .and_then(|ma| Ok((ma, mmap_file(&b_path, "second")?))); + let (ma, mb) = match sides { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")); + }, + }; + let progress = + ma.len() >= PROGRESS_MIN_BYTES || mb.len() >= PROGRESS_MIN_BYTES; + let parsed = parse_side_mmap(&ma, "first", progress) + .and_then(|a| Ok((a, parse_side_mmap(&mb, "second", progress)?))); + let ((ia, ea), (ib, eb)) = match parsed { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")); + }, + }; + match run_diff( + LazySide { env: &ea, index: &ia, data: &ma[..] }, + LazySide { env: &eb, index: &ib, data: &mb[..] }, + want_meta, + progress, + "rs_diff_env_files", + ) { + Ok(d) => LeanIOResult::ok(LeanIxonEnvDiff::build(&d)), + Err(e) => LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")), + } +} + +/// FFI: byte-equality of two files. `(a b : String) → IO Bool`. +/// Length fast path via metadata, then an mmap memcmp — no heap reads. +#[unsafe(no_mangle)] +pub extern "C" fn rs_ixe_files_equal( + a_path: LeanString>, + b_path: LeanString>, +) -> LeanIOResult { + let a_path = a_path.to_string(); + let b_path = b_path.to_string(); + let len = |path: &str, which: &str| -> Result { + std::fs::metadata(path) + .map(|m| m.len()) + .map_err(|e| format!("{which} input {path}: {e}")) + }; + let lens = + len(&a_path, "first").and_then(|la| Ok((la, len(&b_path, "second")?))); + let (la, lb) = match lens { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_ixe_files_equal: {e}")); + }, + }; + let eq = if la != lb { + false + } else if la == 0 { + true // two empty files; mmap of a zero-length file errors + } else { + let maps = mmap_file(&a_path, "first") + .and_then(|ma| Ok((ma, mmap_file(&b_path, "second")?))); + match maps { + Ok((ma, mb)) => ma[..] == mb[..], + Err(e) => { + return LeanIOResult::error_string(&format!("rs_ixe_files_equal: {e}")); + }, + } + }; + LeanIOResult::ok(LeanOwned::box_usize(usize::from(eq))) +} diff --git a/crates/ffi/src/lean_ixon/env.rs b/crates/ffi/src/lean_ixon/env.rs index edac79ad2..39ab800de 100644 --- a/crates/ffi/src/lean_ixon/env.rs +++ b/crates/ffi/src/lean_ixon/env.rs @@ -4,10 +4,10 @@ //! RawConst, RawNamed, RawBlob, RawComm. use crate::lean::{ - LeanIxName, LeanIxonComm, LeanIxonConstant, LeanIxonConstantMeta, - LeanIxonRawBlob, LeanIxonRawComm, LeanIxonRawConst, LeanIxonRawConstSlice, - LeanIxonRawEnv, LeanIxonRawEnvLazy, LeanIxonRawNameEntry, LeanIxonRawNamed, - LeanIxonRawNamedLite, + LeanIxName, LeanIxReducibilityHints, LeanIxonComm, LeanIxonConstant, + LeanIxonConstantMeta, LeanIxonRawBlob, LeanIxonRawComm, LeanIxonRawConst, + LeanIxonRawConstSlice, LeanIxonRawEnv, LeanIxonRawEnvLazy, + LeanIxonRawNameEntry, LeanIxonRawNamed, LeanIxonRawNamedLite, }; use ix_common::address::Address; use ix_common::env::{Name, ReducibilityHints}; @@ -17,7 +17,8 @@ use ixon::env::{Env as IxonEnv, LazyIndex, Named as IxonNamed}; use ixon::merkle::merkle_root_canonical; use ixon::metadata::ConstantMeta; use lean_ffi::object::{ - LeanArray, LeanBorrowed, LeanByteArray, LeanExcept, LeanOwned, LeanRef, + LeanArray, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, LeanOwned, + LeanProd, LeanRef, }; use crate::builder::LeanBuildCache; @@ -226,7 +227,7 @@ impl LeanIxonRawNameEntry { } // ============================================================================= -// RawEnv (consts, named, blobs, comms, names) +// RawEnv (consts, named, blobs, comms, names, main, assumptions, anonHints) // ============================================================================= /// Decoded Ixon.RawEnv @@ -236,6 +237,12 @@ pub struct DecodedRawEnv { pub blobs: Vec, pub comms: Vec, pub names: Vec, + /// Bundle root (`Env::main`). + pub main: Option
, + /// Bundle trust boundary (`Env::assumptions`), sorted ascending. + pub assumptions: Vec
, + /// Explicit reducibility hints (`Env::anon_hints`), sorted by address. + pub anon_hints: Vec<(Address, ReducibilityHints)>, } impl LeanIxonRawEnv { @@ -274,6 +281,27 @@ impl LeanIxonRawEnv { .set(i, LeanIxonRawNameEntry::build(&mut cache, &rn.addr, &rn.name)); } + // Build bundle fields: main (Option Address), assumptions + // (Array Address), anonHints (Array (Address × ReducibilityHints)). + let main_obj: LeanOwned = match &env.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let assumptions_arr = LeanArray::alloc(env.assumptions.len()); + for (i, addr) in env.assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + let hints_arr = LeanArray::alloc(env.anon_hints.len()); + for (i, (addr, hint)) in env.anon_hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + // Build RawEnv structure let ctor = LeanIxonRawEnv::alloc(0); ctor.set_obj(0, consts_arr); @@ -281,10 +309,51 @@ impl LeanIxonRawEnv { ctor.set_obj(2, blobs_arr); ctor.set_obj(3, comms_arr); ctor.set_obj(4, names_arr); + ctor.set_obj(5, main_obj); + ctor.set_obj(6, assumptions_arr); + ctor.set_obj(7, hints_arr); ctor } } +/// Populate the bundle-field constructor slots (5-7: main, +/// assumptions, anonHints) of a `RawEnv` being built field-by-field — +/// the compile FFI assembles its RawEnv manually instead of going +/// through [`LeanIxonRawEnv::build`], and leaving these slots unset is +/// uninitialized memory (segfault on GC). Set-shaped fields are sorted +/// for a deterministic transfer. +pub fn set_raw_env_bundle_fields( + ctor: &LeanIxonRawEnv, + env: &IxonEnv, +) { + let main_obj: LeanOwned = match &env.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let mut assumptions: Vec
= env.assumptions.iter().cloned().collect(); + assumptions.sort_unstable(); + let assumptions_arr = LeanArray::alloc(assumptions.len()); + for (i, addr) in assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + let mut hints: Vec<(Address, ReducibilityHints)> = + env.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); + hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let hints_arr = LeanArray::alloc(hints.len()); + for (i, (addr, hint)) in hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + ctor.set_obj(5, main_obj); + ctor.set_obj(6, assumptions_arr); + ctor.set_obj(7, hints_arr); +} + impl LeanIxonRawEnv { /// Decode Ixon.RawEnv from Lean pointer. pub fn decode(&self) -> DecodedRawEnv { @@ -294,6 +363,23 @@ impl LeanIxonRawEnv { let blobs_arr = ctor.get(2).as_array(); let comms_arr = ctor.get(3).as_array(); let names_arr = ctor.get(4).as_array(); + let main_obj = ctor.get(5); + let assumptions_arr = ctor.get(6).as_array(); + let hints_arr = ctor.get(7).as_array(); + + // `Option Address`: scalar-optimized none, or a tag-0/1 ctor. + let main: Option
= if main_obj.is_scalar() { + None + } else { + let opt = main_obj.as_ctor(); + match opt.tag() { + 0 => None, + 1 => Some( + LeanIxAddress::from_borrowed(opt.get(0).as_byte_array()).decode(), + ), + tag => panic!("Invalid Option tag for RawEnv.main: {tag}"), + } + }; DecodedRawEnv { consts: consts_arr @@ -304,6 +390,17 @@ impl LeanIxonRawEnv { comms: comms_arr.map(|x| LeanIxonRawComm::new(x.to_owned_ref()).decode()), names: names_arr .map(|x| LeanIxonRawNameEntry::new(x.to_owned_ref()).decode()), + main, + assumptions: assumptions_arr + .map(|x| LeanIxAddress::from_borrowed(x.as_byte_array()).decode()), + anon_hints: hints_arr.map(|x| { + let pair = x.as_ctor(); + let addr = + LeanIxAddress::from_borrowed(pair.get(0).as_byte_array()).decode(); + let hint = + LeanIxReducibilityHints::new(pair.get(1).to_owned_ref()).decode(); + (addr, hint) + }), } } } @@ -314,7 +411,7 @@ impl LeanIxonRawEnv { /// Reconstruct a Rust IxonEnv from a DecodedRawEnv. pub fn decoded_to_ixon_env(decoded: &DecodedRawEnv) -> IxonEnv { - let env = IxonEnv::new(); + let mut env = IxonEnv::new(); for rc in &decoded.consts { env.store_const(rc.addr.clone(), rc.constant.clone()); } @@ -331,6 +428,11 @@ pub fn decoded_to_ixon_env(decoded: &DecodedRawEnv) -> IxonEnv { for rc in &decoded.comms { env.store_comm(rc.addr.clone(), rc.comm.clone()); } + env.main = decoded.main.clone(); + env.assumptions.extend(decoded.assumptions.iter().cloned()); + env + .anon_hints + .extend(decoded.anon_hints.iter().map(|(a, h)| (a.clone(), *h))); env } @@ -380,7 +482,22 @@ pub fn ixon_env_to_decoded(env: &IxonEnv) -> Result { name: e.value().clone(), }) .collect(); - Ok(DecodedRawEnv { consts, named, blobs, comms, names }) + // Sort the set-shaped fields so the FFI transfer is deterministic. + let mut assumptions: Vec
= env.assumptions.iter().cloned().collect(); + assumptions.sort_unstable(); + let mut anon_hints: Vec<(Address, ReducibilityHints)> = + env.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); + anon_hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + Ok(DecodedRawEnv { + consts, + named, + blobs, + comms, + names, + main: env.main.clone(), + assumptions, + anon_hints, + }) } // ============================================================================= @@ -451,11 +568,12 @@ pub extern "C" fn rs_de_env( /// FFI: Anonymous-only deserialization (`Env::get_anon`). /// -/// Reads the header + blobs + consts sections; parses and discards -/// the metadata sections (names / named / comms). The returned -/// `RawEnv` has empty `named`, `names`, `comms` arrays. Useful for -/// anon-mode kernel callers that want to avoid the steady-state -/// memory cost of metadata that they will never consult. +/// Reads the header + §1 blobs + §2 consts + §3 anon_hints and stops; +/// the metadata sections (names / named / comms) are laid out after +/// the hints and never touched. The returned `RawEnv` has empty +/// `named`, `names`, `comms` arrays. Useful for anon-mode kernel +/// callers that want to avoid the steady-state memory cost of +/// metadata they will never consult. #[unsafe(no_mangle)] pub extern "C" fn rs_de_env_anon( obj: LeanByteArray>, @@ -490,17 +608,6 @@ pub extern "C" fn rs_de_env_anon( // closure it actually checks. See `Ix.Ixon.deEnvAnon`. // ============================================================================= -/// Encode an optional reducibility hint as `(kind, val)` for the FFI: -/// `0` = none (not a Defn), `1` = Opaque, `2` = Abbrev, `3` = Regular(val). -fn encode_hint(hint: &Option) -> (u64, u64) { - match hint { - None => (0, 0), - Some(ReducibilityHints::Opaque) => (1, 0), - Some(ReducibilityHints::Abbrev) => (2, 0), - Some(ReducibilityHints::Regular(n)) => (3, u64::from(*n)), - } -} - impl LeanIxonRawConstSlice { /// Build `Ixon.RawConstSlice { addr, offset, len }`. pub fn build(addr: &Address, offset: usize, len: usize) -> Self { @@ -513,19 +620,15 @@ impl LeanIxonRawConstSlice { } impl LeanIxonRawNamedLite { - /// Build `Ixon.RawNamedLite { name, addr, hintKind, hintVal }`. + /// Build `Ixon.RawNamedLite { name, addr }`. pub fn build( cache: &mut LeanBuildCache, name: &Name, addr: &Address, - hint: &Option, ) -> Self { - let (kind, val) = encode_hint(hint); let ctor = LeanIxonRawNamedLite::alloc(0); ctor.set_obj(0, LeanIxName::build(cache, name)); ctor.set_obj(1, LeanIxAddress::build(addr)); - ctor.set_num_64(0, kind); - ctor.set_num_64(1, val); ctor } } @@ -541,10 +644,7 @@ fn build_raw_env_lazy(index: &LazyIndex) -> LeanIxonRawEnvLazy { let named_arr = LeanArray::alloc(index.named.len()); for (i, n) in index.named.iter().enumerate() { - named_arr.set( - i, - LeanIxonRawNamedLite::build(&mut cache, &n.name, &n.addr, &n.hint), - ); + named_arr.set(i, LeanIxonRawNamedLite::build(&mut cache, &n.name, &n.addr)); } let blobs_arr = LeanArray::alloc(index.blobs.len()); @@ -552,10 +652,33 @@ fn build_raw_env_lazy(index: &LazyIndex) -> LeanIxonRawEnvLazy { blobs_arr.set(i, LeanIxonRawBlob::build_from_parts(addr, bytes)); } + let main_obj: LeanOwned = match &index.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let assumptions_arr = LeanArray::alloc(index.assumptions.len()); + for (i, addr) in index.assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + + let hints_arr = LeanArray::alloc(index.hints.len()); + for (i, (addr, hint)) in index.hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + let ctor = LeanIxonRawEnvLazy::alloc(0); ctor.set_obj(0, consts_arr); ctor.set_obj(1, named_arr); ctor.set_obj(2, blobs_arr); + ctor.set_obj(3, main_obj); + ctor.set_obj(4, assumptions_arr); + ctor.set_obj(5, hints_arr); ctor } diff --git a/crates/ffi/src/lean_ixon/meta.rs b/crates/ffi/src/lean_ixon/meta.rs index 5c8af7c00..844a68277 100644 --- a/crates/ffi/src/lean_ixon/meta.rs +++ b/crates/ffi/src/lean_ixon/meta.rs @@ -3,9 +3,8 @@ //! Includes: DataValue, KVMap, ExprMetaData, ExprMetaArena, ConstantMeta, Named, Comm use crate::lean::{ - LeanIxReducibilityHints, LeanIxonComm, LeanIxonConstantMeta, - LeanIxonDataValue, LeanIxonExprMetaArena, LeanIxonExprMetaData, - LeanIxonNamed, + LeanIxonComm, LeanIxonConstantMeta, LeanIxonDataValue, LeanIxonExprMetaArena, + LeanIxonExprMetaData, LeanIxonNamed, }; use ix_common::address::Address; use ix_common::env::BinderInfo; @@ -349,7 +348,7 @@ impl LeanIxonConstantMeta { /// | Variant | Tag | Obj fields | Scalar bytes | /// |---------|-----|-----------|-------------| /// | empty | 0 | 0 | 0 | - /// | defn | 1 | 6 (name, lvls, hints, all, ctx, arena) | 16 (2× u64) | + /// | defn | 1 | 5 (name, lvls, all, ctx, arena) | 16 (2× u64) | /// | axio | 2 | 3 (name, lvls, arena) | 8 (1× u64) | /// | quot | 3 | 3 (name, lvls, arena) | 8 (1× u64) | /// | indc | 4 | 6 (name, lvls, ctors, all, ctx, arena) | 8 (1× u64) | @@ -363,7 +362,6 @@ impl LeanIxonConstantMeta { ConstantMetaInfo::Def { name, lvls, - hints, all, ctx, arena, @@ -373,10 +371,9 @@ impl LeanIxonConstantMeta { let ctor = LeanIxonConstantMeta::alloc(1); ctor.set_obj(0, LeanIxAddress::build(name)); ctor.set_obj(1, LeanIxAddress::build_array(lvls)); - ctor.set_obj(2, LeanIxReducibilityHints::build(hints)); - ctor.set_obj(3, LeanIxAddress::build_array(all)); - ctor.set_obj(4, LeanIxAddress::build_array(ctx)); - ctor.set_obj(5, LeanIxonExprMetaArena::build(arena)); + ctor.set_obj(2, LeanIxAddress::build_array(all)); + ctor.set_obj(3, LeanIxAddress::build_array(ctx)); + ctor.set_obj(4, LeanIxonExprMetaArena::build(arena)); ctor.set_num_64(0, *type_root); ctor.set_num_64(1, *value_root); ctor @@ -486,18 +483,15 @@ impl LeanIxonConstantMeta { LeanIxAddress::from_borrowed(self.get_obj(0).as_byte_array()) .decode(); let lvls = decode_address_array(self.get_obj(1).as_array()); - let hints = - LeanIxReducibilityHints::new(self.get_obj(2).to_owned_ref()).decode(); - let all = decode_address_array(self.get_obj(3).as_array()); - let ctx = decode_address_array(self.get_obj(4).as_array()); + let all = decode_address_array(self.get_obj(2).as_array()); + let ctx = decode_address_array(self.get_obj(3).as_array()); let arena = - LeanIxonExprMetaArena::new(self.get_obj(5).to_owned_ref()).decode(); + LeanIxonExprMetaArena::new(self.get_obj(4).to_owned_ref()).decode(); let type_root = self.get_num_64(0); let value_root = self.get_num_64(1); ConstantMeta::new(ConstantMetaInfo::Def { name, lvls, - hints, all, ctx, arena, diff --git a/crates/ffi/src/lean_ixon/pack.rs b/crates/ffi/src/lean_ixon/pack.rs new file mode 100644 index 000000000..d71711034 --- /dev/null +++ b/crates/ffi/src/lean_ixon/pack.rs @@ -0,0 +1,152 @@ +//! `rs_pack_env`: read a serialized env, prune it to the self-contained +//! closure of one named constant ([`ixon::env::Env::prune_to_closure`] +//! semantics — sets `main` and collects reached cut-points into +//! `assumptions`), validate the result +//! ([`ixon::env::Env::validate_closed`]), and write the bundle `.ixe`. +//! +//! The source env is memory-mapped and lazily loaded (constant windows +//! stay zero-copy mmap slices); metadata is never bulk-materialized. +//! The default mode carries display metadata by re-streaming §5 per +//! prune fixpoint round (`Env::prune_to_closure_streaming` — +//! O(survivors) resident metadata); `anon` mode skips metadata +//! entirely (`Env::prune_to_closure_anon` — value closure + §3 hints +//! only, the minimal typecheck/eval artifact). + +use ix_common::address::Address; +use ixon::env::Env as IxonEnv; +use lean_ffi::object::{ + LeanArray, LeanBool, LeanBorrowed, LeanIOResult, LeanOwned, LeanString, +}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use super::diff::mmap_file; + +/// FFI: pack a value bundle. +/// `(envPath mainName : String) → (assume : Array String) → +/// (outPath : String) → (anon verbose : Bool) → IO Unit`. +/// +/// `assume` entries resolve as displayed constant names first, else as +/// 64-hex constant addresses (cut points need not be named). +#[unsafe(no_mangle)] +pub extern "C" fn rs_pack_env( + env_path: LeanString>, + main_name: LeanString>, + assume: LeanArray>, + out_path: LeanString>, + anon: LeanBool>, + verbose: LeanBool>, +) -> LeanIOResult { + let anon = anon.to_bool(); + let verbose = verbose.to_bool(); + let path = env_path.to_string(); + let main_str = main_name.to_string(); + let out = out_path.to_string(); + let assume_vec: Vec = assume.map(|obj| obj.as_string().to_string()); + + let mmap = match mmap_file(&path, "source") { + Ok(m) => m, + Err(e) => return LeanIOResult::error_string(&format!("rs_pack_env: {e}")), + }; + if verbose { + eprintln!( + "[rs_pack_env] parsing {path} ({} MB, lazy reader)...", + mmap.len() / 1_000_000 + ); + } + let (index, names) = match IxonEnv::parse_lazy_index_with_names(&mmap[..]) { + Ok(v) => v, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to index {path}: {e}" + )); + }, + }; + let src = match IxonEnv::from_lazy_index_mmap(&index, &mmap) { + Ok(env) => env, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to load {path}: {e}" + )); + }, + }; + if verbose { + eprintln!( + "[rs_pack_env] source env: {} consts, {} named, {} blobs", + src.consts.len(), + src.named.len(), + src.blobs.len() + ); + } + + // Resolve displayed names → addresses through the lazy index's + // name→addr entries (the `rs_env_extract` idiom). + let by_name: FxHashMap = + index.named.iter().map(|n| (n.name.to_string(), n.addr.clone())).collect(); + let main = match by_name.get(&main_str) { + Some(a) => a.clone(), + None => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: no constant named {main_str} in {path}" + )); + }, + }; + let mut assumed: FxHashSet
= FxHashSet::default(); + let mut unresolved: Vec<&str> = Vec::new(); + for s in &assume_vec { + if let Some(a) = by_name.get(s.as_str()) { + assumed.insert(a.clone()); + } else if let Some(a) = Address::from_hex(s) { + assumed.insert(a); + } else { + unresolved.push(s); + } + } + if !unresolved.is_empty() { + return LeanIOResult::error_string(&format!( + "rs_pack_env: --assume entries neither named in {path} nor 64-hex \ + addresses: [{}]", + unresolved.join(", ") + )); + } + + let bundle = if anon { + src.prune_to_closure_anon(&main, &assumed) + } else { + src.prune_to_closure_streaming(&index, &mmap[..], &names, &main, &assumed) + }; + let bundle = match bundle { + Ok(b) => b, + Err(e) => return LeanIOResult::error_string(&format!("rs_pack_env: {e}")), + }; + if let Err(e) = bundle.validate_closed() { + return LeanIOResult::error_string(&format!("rs_pack_env: {e}")); + } + let mut buf = Vec::new(); + if let Err(e) = bundle.put(&mut buf) { + return LeanIOResult::error_string(&format!( + "rs_pack_env: bundle serialization failed: {e}" + )); + } + if let Err(e) = std::fs::write(&out, &buf) { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to write {out}: {e}" + )); + } + if verbose { + eprintln!("[rs_pack_env] main {} ({main_str})", main.hex()); + eprintln!( + "[rs_pack_env] kept {}/{} consts, {}/{} named, {}/{} blobs, \ + {} assumption(s){}", + bundle.consts.len(), + src.consts.len(), + bundle.named.len(), + src.named.len(), + bundle.blobs.len(), + src.blobs.len(), + bundle.assumptions.len(), + if anon { " [anon: no display metadata]" } else { "" } + ); + eprintln!("[rs_pack_env] wrote {out} ({} bytes)", buf.len()); + } + LeanIOResult::ok(LeanOwned::box_usize(0)) +} diff --git a/crates/ffi/src/lean_ixon/serialize.rs b/crates/ffi/src/lean_ixon/serialize.rs index 68061c03a..ef29caafd 100644 --- a/crates/ffi/src/lean_ixon/serialize.rs +++ b/crates/ffi/src/lean_ixon/serialize.rs @@ -234,6 +234,52 @@ pub extern "C" fn rs_eq_env_serialization( } } + // Bundle header fields. + if rust_env.main != decoded.main { + if debug { + eprintln!( + "[rs_eq_env_serialization] main mismatch: rust={:?}, decoded={:?}", + rust_env.main.as_ref().map(Address::hex), + decoded.main.as_ref().map(Address::hex), + ); + } + return false; + } + let decoded_assumptions: rustc_hash::FxHashSet
= + decoded.assumptions.iter().cloned().collect(); + if rust_env.assumptions != decoded_assumptions { + if debug { + eprintln!( + "[rs_eq_env_serialization] assumptions mismatch: rust={}, decoded={}", + rust_env.assumptions.len(), + decoded_assumptions.len(), + ); + } + return false; + } + + // Hints: `anon_hints` is the single home for hints (the writers + // serialize the map directly), so the parsed env's map must equal + // the decoded RawEnv's hints verbatim. + let expected_hints: rustc_hash::FxHashMap< + Address, + ix_common::env::ReducibilityHints, + > = decoded.anon_hints.iter().cloned().collect(); + let hints_match = rust_env.anon_hints.len() == expected_hints.len() + && expected_hints + .iter() + .all(|(a, h)| rust_env.anon_hints.get(a).map(|r| *r) == Some(*h)); + if !hints_match { + if debug { + eprintln!( + "[rs_eq_env_serialization] anon_hints mismatch: rust={}, expected={}", + rust_env.anon_hints.len(), + expected_hints.len(), + ); + } + return false; + } + true } diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs new file mode 100644 index 000000000..7981cc7af --- /dev/null +++ b/crates/ixon/src/diff.rs @@ -0,0 +1,2505 @@ +//! Structured diff of two environments. +//! +//! [`diff_envs`] compares two [`Env`]s and produces an [`EnvDiff`] report. +//! Content-addressing frames the comparison: a name "changed" iff its +//! `Named.addr` differs, and constant value-equality coincides with +//! address-equality. Because addresses hash the *representation* +//! (sharing/refs/univs tables included), a changed address may carry no +//! semantic field difference — table reordering or different sharing +//! decisions — which the classifier reports honestly as `"encoding"`. +//! +//! Two modes: +//! - anon (`meta = false`, the default): only anonymous structure is +//! compared — name→addr changes (with per-field classification of the +//! two constants), consts/blobs set differences, comms, `main`, +//! `assumptions`, and reducibility hints (which live in the anon §3 +//! section and drive kernel unfolding). Names serve as join/display +//! keys only. +//! - meta (`meta = true`): additionally compares `Named.meta` / +//! `Named.original` content, populating [`EnvDiff::named_meta_only`] +//! and [`NamedChange::meta_fields`]. +//! +//! Every changed row additionally carries a ripple verdict +//! ([`NamedChange::rippled`]): after the join, changed pairs are +//! re-classified under a quotient where an (old, new) address pair +//! compares equal when some changed name maps old→new. Rows whose +//! residual labels are all `"encoding"`/`"block-siblings"` are +//! *rippled* — fully explained by dependency re-addressing (the +//! content-address ripple: one edited constant re-addresses its whole +//! reverse-dependency cone); the rest are *roots* (intrinsic edits). A +//! single-level mapping is complete because expression comparison only +//! ever consults immediate-dependency addresses — composition across +//! the DAG happens through the per-row verdicts, never through the map. +//! Accepted semantic edges: +//! - Induced re-elaboration verdicts root: a dependency's +//! universe/arity signature change alters dependents' terms beyond +//! addresses (e.g. `Ref` univ-argument arity), so "roots" +//! over-approximates "human edits" when signatures change. +//! - An intrinsic edit of a block member with no named projection has +//! no root row of its own (blocks are unnamed; in practice ix names +//! every projection, so the member's projection row is the root). +//! - The `"block"` fallback (block bytes missing, e.g. behind a +//! bundle's assumption cut) verdicts root — fail-safe over-report. +//! +//! Expression comparison ([`exprs_equal`]) is a lockstep structural walk +//! that resolves table indices through each side's own `Constant` tables: +//! identical expr bytes over different tables compare *different*, and +//! different indices resolving to the same addresses with the same +//! structure compare *equal*. +//! +//! Out of scope: `Env.names` (hash-consed name components — derived data; +//! the named join subsumes user-visible changes) and explanations for +//! orphan consts (mutual blocks and aux originals appear in the consts +//! set difference as raw truth). Synthetic Muts names (`Ix.` +//! prefixed) churn as one removed+added pair per changed block; display +//! layers may group them. + +use std::cmp::Ordering; +use std::sync::Arc; + +use rustc_hash::{FxHashMap, FxHashSet}; + +use ix_common::address::Address; +use ix_common::env::{Name, ReducibilityHints}; + +use super::constant::{ + Constant, ConstantInfo, Constructor, Definition, Inductive, MutConst, + Recursor, +}; +use super::env::{Env, LazyIndex, Named}; +use super::expr::Expr; +use super::serialize::NamedMetaCursor; +use super::univ::Univ; + +/// Per-env entity counts, reported for both inputs so display layers can +/// print a header without re-walking the envs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct EnvStats { + pub consts: usize, + pub named: usize, + pub blobs: usize, + pub comms: usize, +} + +/// One name present in both envs whose constant address changed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedChange { + /// `Name::pretty()` of the joined name. + pub name: String, + pub old_addr: Address, + pub new_addr: Address, + /// [`kind_label`] of each side's `ConstantInfo` variant. + pub old_kind: &'static str, + pub new_kind: &'static str, + /// Changed-field labels, e.g. `"value"`, `"block.ctors[0].type"`. + /// Never empty: `"kind"` when the variants differ, `"encoding"` when + /// no semantic field difference explains the address change. + pub fields: Vec, + /// Metadata component labels when the metadata ALSO changed. Only + /// populated in meta mode; may be empty. + pub meta_fields: Vec, + /// True iff the address change is fully explained by dependency + /// re-addressing: re-classified under the old→new quotient of all + /// changed rows, the residual labels are all `"encoding"` / + /// `"block-siblings"`. `fields` stays the strict (unquotiented) + /// classification; this is the orthogonal root-vs-ripple verdict. + pub rippled: bool, +} + +/// Report produced by [`diff_envs`]. All name-keyed vectors are sorted +/// by (pretty name, name hash); address vectors are sorted ascending. +/// Set-difference lists are complete — display layers cap as needed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EnvDiff { + /// `None` = unchanged, otherwise `(a.main, b.main)`. + pub main_changed: Option<(Option
, Option
)>, + pub assumptions_added: Vec
, + pub assumptions_removed: Vec
, + /// Names only in the second env, with their constant address. + pub named_added: Vec<(String, Address)>, + /// Names only in the first env, with their constant address. + pub named_removed: Vec<(String, Address)>, + pub named_changed: Vec, + /// Same constant address, different metadata (meta mode only). + pub named_meta_only: Vec<(String, Vec)>, + pub comms_added: Vec
, + pub comms_removed: Vec
, + /// Same commitment address, different `Comm` payload (possible + /// because the comms section is not hash-verified on load). + pub comms_changed: Vec
, + pub consts_only_a: Vec
, + pub consts_only_b: Vec
, + pub blobs_only_a: Vec
, + pub blobs_only_b: Vec
, + /// Hint deltas for constants present in BOTH envs (hint deltas of + /// added/removed constants are implied by the other sections). + /// Rendered as `"opaque" | "abbrev" | "regular(N)" | "none"`. + pub hints_changed: Vec<(Address, String, String)>, + pub stats_a: EnvStats, + pub stats_b: EnvStats, +} + +impl EnvDiff { + /// True when no difference was found. Ignores `stats_*` (always + /// populated). + pub fn is_empty(&self) -> bool { + self.main_changed.is_none() + && self.assumptions_added.is_empty() + && self.assumptions_removed.is_empty() + && self.named_added.is_empty() + && self.named_removed.is_empty() + && self.named_changed.is_empty() + && self.named_meta_only.is_empty() + && self.comms_added.is_empty() + && self.comms_removed.is_empty() + && self.comms_changed.is_empty() + && self.consts_only_a.is_empty() + && self.consts_only_b.is_empty() + && self.blobs_only_a.is_empty() + && self.blobs_only_b.is_empty() + && self.hints_changed.is_empty() + } +} + +/// Short label for a `ConstantInfo` variant. +pub fn kind_label(info: &ConstantInfo) -> &'static str { + match info { + ConstantInfo::Defn(_) => "defn", + ConstantInfo::Recr(_) => "recr", + ConstantInfo::Axio(_) => "axio", + ConstantInfo::Quot(_) => "quot", + ConstantInfo::CPrj(_) => "cprj", + ConstantInfo::RPrj(_) => "rprj", + ConstantInfo::IPrj(_) => "iprj", + ConstantInfo::DPrj(_) => "dprj", + ConstantInfo::Muts(_) => "muts", + } +} + +// ============================================================================ +// Expression comparison +// ============================================================================ + +/// Follow a `Share` chain through `c.sharing` until a non-`Share` node. +/// Hop budget guards corrupt cycles (honest constants only reference +/// strictly earlier sharing entries, so chains are finite). +fn resolve_share<'c>( + mut e: &'c Expr, + c: &'c Constant, + side: &str, +) -> Result<&'c Expr, String> { + let mut hops: usize = 0; + while let Expr::Share(i) = e { + hops += 1; + if hops > c.sharing.len() + 1 { + return Err(format!("diff: Share cycle detected on the {side} side")); + } + let idx = usize::try_from(*i).map_err(|_e| { + format!("diff: Share index {i} overflows usize on the {side} side") + })?; + e = c.sharing.get(idx).map(|a| a.as_ref()).ok_or_else(|| { + format!( + "diff: Share index {i} out of bounds ({} sharing entries) on the {side} side", + c.sharing.len() + ) + })?; + } + Ok(e) +} + +fn univ_at<'c>( + c: &'c Constant, + idx: u64, + side: &str, +) -> Result<&'c Univ, String> { + usize::try_from(idx) + .ok() + .and_then(|i| c.univs.get(i)) + .map(|u| u.as_ref()) + .ok_or_else(|| { + format!( + "diff: univ index {idx} out of bounds ({} entries) on the {side} side", + c.univs.len() + ) + }) +} + +fn ref_at<'c>( + c: &'c Constant, + idx: u64, + side: &str, +) -> Result<&'c Address, String> { + usize::try_from(idx).ok().and_then(|i| c.refs.get(i)).ok_or_else(|| { + format!( + "diff: ref index {idx} out of bounds ({} entries) on the {side} side", + c.refs.len() + ) + }) +} + +/// Old→new address pairs harvested from every changed named row in +/// pass 1 (kind-change and encoding rows included). Set-valued to +/// tolerate name splits: two names sharing an old address, diverging in +/// the new env. Directional — keys are A-side (old) addresses. +type AddrMap = FxHashMap>; + +/// Lockstep expression comparator over one pair of table contexts. +/// +/// The pointer-pair memo persists across `eq` calls for the same +/// constant pair (tables are constant-level, so shared subterms recur +/// across type/value/rules). Entries record "heads matched, children +/// enqueued" — an unequal result returns early and leaves enqueued +/// children unverified, so [`Self::eq`] clears the memo on `false`. +/// +/// `map` selects the address equality: `None` = strict (pass 1), +/// `Some` = the ripple quotient (pass 2). +struct ExprCmp<'a> { + ca: &'a Constant, + cb: &'a Constant, + memo: FxHashSet<(usize, usize)>, + map: Option<&'a AddrMap>, +} + +impl<'a> ExprCmp<'a> { + fn new(ca: &'a Constant, cb: &'a Constant, map: Option<&'a AddrMap>) -> Self { + ExprCmp { ca, cb, memo: FxHashSet::default(), map } + } + + /// Address equality under the selected mode. `l` MUST be the A-side + /// (old) address and `r` the B-side (new) — the quotient map is + /// directional. Every comparison site preserves that order (the walk + /// always pairs (A-expr, B-expr)). + fn addrs_eq(&self, l: &Address, r: &Address) -> bool { + l == r || self.map.is_some_and(|m| m.get(l).is_some_and(|v| v.contains(r))) + } + + fn eq(&mut self, a: &'a Expr, b: &'a Expr) -> Result { + let r = self.eq_inner(a, b)?; + if !r { + self.memo.clear(); + } + Ok(r) + } + + fn eq_inner(&mut self, a: &'a Expr, b: &'a Expr) -> Result { + // Iterative walk: kernel terms nest thousands deep, never recurse. + let mut stack: Vec<(&'a Expr, &'a Expr)> = vec![(a, b)]; + while let Some((ea, eb)) = stack.pop() { + let ea = resolve_share(ea, self.ca, "left")?; + let eb = resolve_share(eb, self.cb, "right")?; + let key = + (std::ptr::from_ref(ea) as usize, std::ptr::from_ref(eb) as usize); + if !self.memo.insert(key) { + continue; + } + match (ea, eb) { + (Expr::Var(x), Expr::Var(y)) => { + if x != y { + return Ok(false); + } + }, + (Expr::Sort(i), Expr::Sort(j)) => { + if univ_at(self.ca, *i, "left")? != univ_at(self.cb, *j, "right")? { + return Ok(false); + } + }, + (Expr::Ref(i, us), Expr::Ref(j, vs)) => { + if !self.addrs_eq( + ref_at(self.ca, *i, "left")?, + ref_at(self.cb, *j, "right")?, + ) || !self.univ_lists_equal(us, vs)? + { + return Ok(false); + } + }, + (Expr::Rec(i, us), Expr::Rec(j, vs)) => { + // Intra-block member index: positional, compare raw. + if i != j || !self.univ_lists_equal(us, vs)? { + return Ok(false); + } + }, + (Expr::Str(i), Expr::Str(j)) | (Expr::Nat(i), Expr::Nat(j)) => { + // Blob content behind equal addresses is identical by + // content-addressing (blob sections are hash-verified on load). + // Blob addresses never enter the quotient map (it holds + // constant addresses from named rows), so literal changes stay + // intrinsic under pass 2 — `addrs_eq` degenerates to `==`. + if !self.addrs_eq( + ref_at(self.ca, *i, "left")?, + ref_at(self.cb, *j, "right")?, + ) { + return Ok(false); + } + }, + (Expr::Prj(ti, fi, va), Expr::Prj(tj, fj, vb)) => { + if fi != fj + || !self.addrs_eq( + ref_at(self.ca, *ti, "left")?, + ref_at(self.cb, *tj, "right")?, + ) + { + return Ok(false); + } + stack.push((va, vb)); + }, + (Expr::App(f1, x1), Expr::App(f2, x2)) + | (Expr::Lam(f1, x1), Expr::Lam(f2, x2)) + | (Expr::All(f1, x1), Expr::All(f2, x2)) => { + stack.push((f1, f2)); + stack.push((x1, x2)); + }, + (Expr::Let(n1, t1, v1, b1), Expr::Let(n2, t2, v2, b2)) => { + if n1 != n2 { + return Ok(false); + } + stack.push((t1, t2)); + stack.push((v1, v2)); + stack.push((b1, b2)); + }, + _ => return Ok(false), + } + } + Ok(true) + } + + fn univ_lists_equal(&self, us: &[u64], vs: &[u64]) -> Result { + if us.len() != vs.len() { + return Ok(false); + } + for (i, j) in us.iter().zip(vs.iter()) { + // Univ trees are self-contained (Univ::Var is a universe-parameter + // de Bruijn index, not a table index), so deep PartialEq is right. + if univ_at(self.ca, *i, "left")? != univ_at(self.cb, *j, "right")? { + return Ok(false); + } + } + Ok(true) + } +} + +/// Structural equality of two expressions, resolving `Share`/`Sort`/ +/// `Ref`/`Rec`-univ/`Str`/`Nat`/`Prj` indices through each side's own +/// `Constant` tables. Errors on out-of-bounds indices or `Share` cycles +/// (corrupt constants). +pub fn exprs_equal( + a: &Expr, + ca: &Constant, + b: &Expr, + cb: &Constant, +) -> Result { + ExprCmp::new(ca, cb, None).eq(a, b) +} + +// ============================================================================ +// Per-kind field classification +// ============================================================================ + +fn classify_defn<'x>( + a: &'x Definition, + b: &'x Definition, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.kind != b.kind { + out.push(format!("{prefix}def-kind")); + } + if a.safety != b.safety { + out.push(format!("{prefix}safety")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if !cmp.eq(&a.value, &b.value)? { + out.push(format!("{prefix}value")); + } + Ok(()) +} + +fn classify_recr<'x>( + a: &'x Recursor, + b: &'x Recursor, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.k != b.k { + out.push(format!("{prefix}k")); + } + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.indices != b.indices { + out.push(format!("{prefix}indices")); + } + if a.motives != b.motives { + out.push(format!("{prefix}motives")); + } + if a.minors != b.minors { + out.push(format!("{prefix}minors")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if a.rules.len() != b.rules.len() { + out.push(format!("{prefix}rules.len")); + } + for (i, (ra, rb)) in a.rules.iter().zip(b.rules.iter()).enumerate() { + if ra.fields != rb.fields { + out.push(format!("{prefix}rules[{i}].fields")); + } + if !cmp.eq(&ra.rhs, &rb.rhs)? { + out.push(format!("{prefix}rules[{i}].rhs")); + } + } + Ok(()) +} + +fn classify_ctor<'x>( + a: &'x Constructor, + b: &'x Constructor, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.cidx != b.cidx { + out.push(format!("{prefix}cidx")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.fields != b.fields { + out.push(format!("{prefix}fields")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + Ok(()) +} + +fn classify_indc<'x>( + a: &'x Inductive, + b: &'x Inductive, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.indices != b.indices { + out.push(format!("{prefix}indices")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if a.ctors.len() != b.ctors.len() { + out.push(format!("{prefix}ctors.len")); + } + for (j, (ca, cb)) in a.ctors.iter().zip(b.ctors.iter()).enumerate() { + classify_ctor(ca, cb, cmp, &format!("{prefix}ctors[{j}]."), out)?; + } + Ok(()) +} + +fn classify_mut_member<'x>( + a: &'x MutConst, + b: &'x MutConst, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + match (a, b) { + (MutConst::Defn(x), MutConst::Defn(y)) => { + classify_defn(x, y, cmp, prefix, out) + }, + (MutConst::Indc(x), MutConst::Indc(y)) => { + classify_indc(x, y, cmp, prefix, out) + }, + (MutConst::Recr(x), MutConst::Recr(y)) => { + classify_recr(x, y, cmp, prefix, out) + }, + _ => { + out.push(format!("{prefix}kind")); + Ok(()) + }, + } +} + +fn classify_muts<'x>( + ma: &'x [MutConst], + mb: &'x [MutConst], + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if ma.len() != mb.len() { + out.push(format!("{prefix}members.len")); + } + for (i, (a, b)) in ma.iter().zip(mb.iter()).enumerate() { + classify_mut_member(a, b, cmp, &format!("{prefix}members[{i}]."), out)?; + } + Ok(()) +} + +/// Same-kind projection pair whose `block` differs: descend one level +/// into the two Muts blocks and classify the projected member, with +/// labels prefixed `block.`. A member identical under resolved +/// comparison means the block hash moved because a *sibling* changed — +/// reported as `"block-siblings"`. Any structural surprise (block +/// missing — e.g. behind a bundle's assumption cut —, unparseable, not +/// Muts, index out of range) falls back to the honest `"block"` label. +/// No deeper recursion: blocks do not nest. +fn classify_prj( + env_a: &Env, + env_b: &Env, + idx_a: u64, + idx_b: u64, + cidx: Option<(u64, u64)>, + block_a: &Address, + block_b: &Address, + map: Option<&AddrMap>, + out: &mut Vec, +) -> Result<(), String> { + if idx_a != idx_b { + out.push("idx".to_string()); + } + if let Some((ca, cb)) = cidx + && ca != cb + { + out.push("cidx".to_string()); + } + // Strict `==` deliberately, even under the quotient: blocks are + // unnamed (their synthetic names embed the hash, so they never join + // as changed rows and never enter the map), and descent is mandatory + // — a block-internal intrinsic edit has no named row of its own, so + // short-circuiting a "mapped" block pair would hide the root. + if block_a == block_b { + return Ok(()); + } + // With differing member coordinates the two projections target + // different members; per-member block detail would be meaningless. + if idx_a != idx_b || cidx.is_some_and(|(x, y)| x != y) { + out.push("block".to_string()); + return Ok(()); + } + let (Some(Ok(ba)), Some(Ok(bb))) = + (env_a.try_get_const(block_a), env_b.try_get_const(block_b)) + else { + out.push("block".to_string()); + return Ok(()); + }; + let (ConstantInfo::Muts(ma), ConstantInfo::Muts(mb)) = (&ba.info, &bb.info) + else { + out.push("block".to_string()); + return Ok(()); + }; + let Some(idx) = usize::try_from(idx_a).ok() else { + out.push("block".to_string()); + return Ok(()); + }; + let (Some(mem_a), Some(mem_b)) = (ma.get(idx), mb.get(idx)) else { + out.push("block".to_string()); + return Ok(()); + }; + // Expression tables are constant-level: members resolve through the + // enclosing block constants' tables. + let mut cmp = ExprCmp::new(&ba, &bb, map); + let before = out.len(); + match cidx { + None => classify_mut_member(mem_a, mem_b, &mut cmp, "block.", out)?, + Some((cx, _)) => { + let (MutConst::Indc(ia), MutConst::Indc(ib)) = (mem_a, mem_b) else { + out.push("block".to_string()); + return Ok(()); + }; + let Some(ci) = usize::try_from(cx).ok() else { + out.push("block".to_string()); + return Ok(()); + }; + let (Some(ctor_a), Some(ctor_b)) = (ia.ctors.get(ci), ib.ctors.get(ci)) + else { + out.push("block".to_string()); + return Ok(()); + }; + classify_ctor(ctor_a, ctor_b, &mut cmp, "block.ctor.", out)?; + }, + } + if out.len() == before { + out.push("block-siblings".to_string()); + } + Ok(()) +} + +/// Compare two same-name constants field by field; returns changed-field +/// labels. Never returns an empty list: kind mismatches yield `"kind"`, +/// and an address change with no detected semantic difference yields +/// `"encoding"` (table reorder / sharing-decision churn). +/// +/// `map = None` is the strict pass-1 classification; `Some` re-runs it +/// under the ripple quotient (pass 2), where the labels are consumed +/// only by the [`rippled_labels`] verdict and then discarded. +fn classify_constants( + env_a: &Env, + ca: &Constant, + env_b: &Env, + cb: &Constant, + map: Option<&AddrMap>, +) -> Result, String> { + let mut out = Vec::new(); + let mut cmp = ExprCmp::new(ca, cb, map); + match (&ca.info, &cb.info) { + (ConstantInfo::Defn(a), ConstantInfo::Defn(b)) => { + classify_defn(a, b, &mut cmp, "", &mut out)?; + }, + (ConstantInfo::Recr(a), ConstantInfo::Recr(b)) => { + classify_recr(a, b, &mut cmp, "", &mut out)?; + }, + (ConstantInfo::Axio(a), ConstantInfo::Axio(b)) => { + if a.is_unsafe != b.is_unsafe { + out.push("unsafe".to_string()); + } + if a.lvls != b.lvls { + out.push("lvls".to_string()); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push("type".to_string()); + } + }, + (ConstantInfo::Quot(a), ConstantInfo::Quot(b)) => { + if a.kind != b.kind { + out.push("quot-kind".to_string()); + } + if a.lvls != b.lvls { + out.push("lvls".to_string()); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push("type".to_string()); + } + }, + (ConstantInfo::DPrj(a), ConstantInfo::DPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, + )?; + }, + (ConstantInfo::IPrj(a), ConstantInfo::IPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, + )?; + }, + (ConstantInfo::RPrj(a), ConstantInfo::RPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, + )?; + }, + (ConstantInfo::CPrj(a), ConstantInfo::CPrj(b)) => { + classify_prj( + env_a, + env_b, + a.idx, + b.idx, + Some((a.cidx, b.cidx)), + &a.block, + &b.block, + map, + &mut out, + )?; + }, + (ConstantInfo::Muts(ma), ConstantInfo::Muts(mb)) => { + classify_muts(ma, mb, &mut cmp, "", &mut out)?; + }, + _ => out.push("kind".to_string()), + } + if out.is_empty() { + out.push("encoding".to_string()); + } + Ok(out) +} + +// ============================================================================ +// Metadata comparison (meta mode) +// ============================================================================ + +/// Component labels for a same-addr metadata difference. +fn meta_component_labels(a: &Named, b: &Named) -> Vec { + let mut out = Vec::new(); + let (am, bm) = (a.meta(), b.meta()); + if am.info != bm.info { + let (ka, kb) = (am.info.kind_name(), bm.info.kind_name()); + if ka == kb { + out.push("meta.info".to_string()); + } else { + out.push(format!("meta.info({ka}→{kb})")); + } + } + if am.meta_sharing != bm.meta_sharing { + out.push("meta.sharing".to_string()); + } + if am.meta_refs != bm.meta_refs { + out.push("meta.refs".to_string()); + } + if am.meta_univs != bm.meta_univs { + out.push("meta.univs".to_string()); + } + match (a.original(), b.original()) { + (None, None) => {}, + (None, Some(_)) => out.push("original.added".to_string()), + (Some(_), None) => out.push("original.removed".to_string()), + (Some((aa, ma)), Some((ab, mb))) => { + if aa != ab { + out.push("original.addr".to_string()); + } + if ma != mb { + out.push("original.meta".to_string()); + } + }, + } + out +} + +// ============================================================================ +// Env diff +// ============================================================================ + +fn hint_label(h: Option<&ReducibilityHints>) -> String { + match h { + None => "none".to_string(), + Some(ReducibilityHints::Opaque) => "opaque".to_string(), + Some(ReducibilityHints::Abbrev) => "abbrev".to_string(), + Some(ReducibilityHints::Regular(n)) => format!("regular({n})"), + } +} + +fn env_stats(e: &Env) -> EnvStats { + EnvStats { + consts: e.consts.len(), + named: e.named.len(), + blobs: e.blobs.len(), + comms: e.comms.len(), + } +} + +/// Materialize the constant behind a named row's address, with +/// name-and-side error context. Shared by the pass-1 classification and +/// the pass-2 ripple verdict (constants are re-parsed on every call — +/// `LazyConstant` deliberately keeps no parse cache, which is what +/// bounds resident memory at mathlib scale). +fn materialize_const( + env: &Env, + addr: &Address, + name: &Name, + which: &str, +) -> Result, String> { + env + .try_get_const(addr) + .ok_or_else(|| { + format!( + "diff: named '{}' addr {} not present in consts of the {which} env", + name.pretty(), + addr.hex() + ) + })? + .map_err(|e| { + format!("diff: named '{}' ({}): {e}", name.pretty(), addr.hex()) + }) +} + +fn classify_named_change( + env_a: &Env, + env_b: &Env, + name: &Name, + na: &Named, + nb: &Named, + meta_fields: Vec, +) -> Result { + let ca = materialize_const(env_a, &na.addr, name, "first")?; + let cb = materialize_const(env_b, &nb.addr, name, "second")?; + let fields = classify_constants(env_a, &ca, env_b, &cb, None)?; + Ok(NamedChange { + name: name.pretty(), + old_addr: na.addr.clone(), + new_addr: nb.addr.clone(), + old_kind: kind_label(&ca.info), + new_kind: kind_label(&cb.info), + fields, + meta_fields, + // Verdict assigned by pass 2 (a row is a root until explained). + rippled: false, + }) +} + +/// Pass-2 verdict over quotient labels: rippled iff every residual +/// label is explained by re-addressing — `"encoding"` (quotient-equal +/// everywhere; the fallback when no field label fired) or +/// `"block-siblings"` (the projected member is quotient-equal; the +/// block hash moved for reasons surfaced by other rows). Scalar labels, +/// `"kind"`, coordinate labels (`"idx"`/`"cidx"`), `"block"`, and all +/// `block.*` detail labels mean intrinsic change → root. Labels are +/// never empty (the `"encoding"` fallback), so `all` cannot pass +/// vacuously. +fn rippled_labels(labels: &[String]) -> bool { + labels.iter().all(|l| l == "encoding" || l == "block-siblings") +} + +/// Sort by (pretty name, name hash) — the hash tiebreak keeps distinct +/// names with equal pretty forms deterministic. +fn sort_by_name(v: &mut [(String, Name, T)]) { + v.sort_by(|x, y| x.0.cmp(&y.0).then_with(|| x.1.cmp(&y.1))); +} + +/// Which diff pass a [`JoinProgress`] event reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DiffPhase { + /// Meta mode only, before the join: the streaming §5 metadata sweep + /// (lockstep merge-join of both files' named sections). + MetaSweep, + /// Pass 1: the named join (change detection + strict classification). + NamedJoin, + /// Pass 2: re-classification of changed rows under the ripple + /// quotient (root vs rippled verdicts). + RippleClassify, +} + +/// Coarse progress for the passes whose cost scales with entry counts. +/// Parses happen before [`diff_envs`] and are the caller's to report. +#[derive(Clone, Copy, Debug)] +pub struct JoinProgress { + /// The pass this event reports. `done` counts reset between phases. + pub phase: DiffPhase, + /// [`DiffPhase::MetaSweep`]: §5 entries merged (max of both sides). + /// [`DiffPhase::NamedJoin`]: first-env named entries processed. + /// [`DiffPhase::RippleClassify`]: changed rows verdicted. + pub done: usize, + /// Total entries of the reporting phase. + pub total: usize, + /// [`DiffPhase::MetaSweep`]: names with differing metadata so far. + /// [`DiffPhase::NamedJoin`]: changed names classified so far. + /// [`DiffPhase::RippleClassify`]: roots found so far. + pub changed: usize, +} + +/// Named-join / meta-sweep entries between progress callbacks. +const JOIN_PROGRESS_STRIDE: usize = 100_000; + +/// Changed rows between pass-2 progress callbacks — one event every +/// ~1-2 s at mathlib scale (each row re-materializes and re-walks a +/// constant pair, so rows are ~10× slower than join probes). +const RIPPLE_PROGRESS_STRIDE: usize = 10_000; + +/// Where metadata comparisons come from. +/// +/// The full-reader path holds complete `Named` values in the envs +/// (`InEnv`); the memory-lean bytes/mmap path never materializes them +/// in bulk, so the streaming §5 sweep precomputes labels per name +/// (`Sweep`); anon mode compares no metadata at all (`None`). +enum MetaSource<'m> { + None, + InEnv, + Sweep(&'m FxHashMap>), +} + +impl MetaSource<'_> { + fn labels(&self, name: &Name, na: &Named, nb: &Named) -> Vec { + match self { + MetaSource::None => Vec::new(), + MetaSource::InEnv => meta_component_labels(na, nb), + MetaSource::Sweep(m) => m.get(name).cloned().unwrap_or_default(), + } + } +} + +/// Diff two environments. `meta = false` (anon mode, the default) +/// compares only anonymous structure; `meta = true` additionally +/// compares `Named` metadata content held in the envs (full-reader +/// path — for the memory-lean bytes path see [`diff_env_bytes`]). See +/// the module docs. +pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { + diff_envs_with(a, b, meta, &mut |_| {}) +} + +/// [`diff_envs`] with a progress callback: `on_progress` fires every +/// `JOIN_PROGRESS_STRIDE` named-join entries and once when each phase +/// completes (final events always carry `done == total`). +pub fn diff_envs_with( + a: &Env, + b: &Env, + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + let source = if meta { MetaSource::InEnv } else { MetaSource::None }; + diff_envs_impl(a, b, &source, on_progress) +} + +/// One side of a lazy-path diff: the env built from `index` over `data` +/// (via [`Env::from_lazy_index`] or [`Env::from_lazy_index_mmap`]). +#[derive(Clone, Copy)] +pub struct LazySide<'a> { + pub env: &'a Env, + pub index: &'a LazyIndex, + pub data: &'a [u8], +} + +/// Memory-lean bytes-level diff (the `ix diff` path): lazy-index +/// structural load plus, in meta mode, the streaming §5 metadata sweep +/// — `ConstantMeta` is never bulk-materialized on either side. +pub fn diff_env_bytes( + a: &[u8], + b: &[u8], + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + let ia = Env::parse_lazy_index(a)?; + let ib = Env::parse_lazy_index(b)?; + let ea = Env::from_lazy_index(&ia, a)?; + let eb = Env::from_lazy_index(&ib, b)?; + diff_envs_lazy( + LazySide { env: &ea, index: &ia, data: a }, + LazySide { env: &eb, index: &ib, data: b }, + meta, + on_progress, + ) +} + +/// Core of the bytes/mmap paths: both sides already lazily loaded. In +/// meta mode, runs the streaming §5 sweep first (metadata labels per +/// name, parse-compare-drop), then the structural join consuming them. +pub fn diff_envs_lazy( + a: LazySide<'_>, + b: LazySide<'_>, + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + if meta { + let map = sweep_meta(&a, &b, on_progress)?; + diff_envs_impl(a.env, b.env, &MetaSource::Sweep(&map), on_progress) + } else { + diff_envs_impl(a.env, b.env, &MetaSource::None, on_progress) + } +} + +/// Streaming §5 metadata sweep: lockstep merge-join of both files' +/// named sections (canonical ascending name-hash order — exactly +/// `Name`'s `Ord`), parsing each side's entry against its own §4 +/// reverse index, comparing, and dropping. Returns component labels +/// for every joined name whose metadata differs — the addr-equal ones +/// become `named_meta_only` rows, the addr-changed ones `meta_fields`. +fn sweep_meta( + a: &LazySide<'_>, + b: &LazySide<'_>, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result>, String> { + let mut cur_a = NamedMetaCursor::open(a.data, a.index)?; + let mut cur_b = NamedMetaCursor::open(b.data, b.index)?; + let an = &a.index.named; + let bn = &b.index.named; + let total = an.len().max(bn.len()); + let mut out: FxHashMap> = FxHashMap::default(); + let (mut i, mut j) = (0usize, 0usize); + let mut fired = 0usize; + while i < an.len() && j < bn.len() { + match an[i].name.cmp(&bn[j].name) { + Ordering::Less => { + cur_a.next_entry()?; + i += 1; + }, + Ordering::Greater => { + cur_b.next_entry()?; + j += 1; + }, + Ordering::Equal => { + let (_, na) = cur_a + .next_entry()? + .ok_or("sweep_meta: first cursor exhausted early")?; + let (_, nb) = cur_b + .next_entry()? + .ok_or("sweep_meta: second cursor exhausted early")?; + // Cursor and index walked the same section — desync means a bug. + if na.addr != an[i].addr || nb.addr != bn[j].addr { + return Err("sweep_meta: cursor desynced from index".to_string()); + } + let labels = meta_component_labels(&na, &nb); + if !labels.is_empty() { + out.insert(an[i].name.clone(), labels); + } + i += 1; + j += 1; + }, + } + let done = i.max(j); + if done / JOIN_PROGRESS_STRIDE > fired && done < total { + fired = done / JOIN_PROGRESS_STRIDE; + on_progress(JoinProgress { + phase: DiffPhase::MetaSweep, + done, + total, + changed: out.len(), + }); + } + } + // Unjoined tails carry no comparison work — cursors just drop. + on_progress(JoinProgress { + phase: DiffPhase::MetaSweep, + done: total, + total, + changed: out.len(), + }); + Ok(out) +} + +fn diff_envs_impl( + a: &Env, + b: &Env, + source: &MetaSource<'_>, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + let mut d = EnvDiff { + stats_a: env_stats(a), + stats_b: env_stats(b), + ..EnvDiff::default() + }; + + // Named join on Name. Alpha-equivalent multi-names (several names + // sharing one constant address) need no special care: each name is + // its own row. + let mut added: Vec<(String, Name, Address)> = Vec::new(); + let mut removed: Vec<(String, Name, Address)> = Vec::new(); + let mut changed: Vec<(String, Name, NamedChange)> = Vec::new(); + let mut meta_only: Vec<(String, Name, Vec)> = Vec::new(); + // Old→new mapping over every changed row, consumed by pass 2. + let mut addr_map: AddrMap = FxHashMap::default(); + let join_total = a.named.len(); + let mut join_done: usize = 0; + for entry in a.named.iter() { + let (name, na) = (entry.key(), entry.value()); + match b.named.get(name) { + None => removed.push((name.pretty(), name.clone(), na.addr.clone())), + Some(nb_ref) => { + let nb = nb_ref.value(); + if na.addr == nb.addr { + let labels = source.labels(name, na, nb); + if !labels.is_empty() { + meta_only.push((name.pretty(), name.clone(), labels)); + } + } else { + let meta_fields = source.labels(name, na, nb); + let change = classify_named_change(a, b, name, na, nb, meta_fields)?; + let slot = addr_map.entry(na.addr.clone()).or_default(); + if !slot.contains(&nb.addr) { + slot.push(nb.addr.clone()); + } + changed.push((name.pretty(), name.clone(), change)); + } + }, + } + join_done += 1; + if join_done.is_multiple_of(JOIN_PROGRESS_STRIDE) && join_done < join_total + { + on_progress(JoinProgress { + phase: DiffPhase::NamedJoin, + done: join_done, + total: join_total, + changed: changed.len(), + }); + } + } + on_progress(JoinProgress { + phase: DiffPhase::NamedJoin, + done: join_total, + total: join_total, + changed: changed.len(), + }); + + // Pass 2 (ripple): re-classify each changed pair under the quotient + // of the just-completed old→new mapping. A pair whose residual labels + // are all explained by re-addressing is rippled (induced by its + // dependencies' address changes); the rest are roots. Alias rows — + // several names spanning one (old, new) pair — share one cached + // verdict and one computation. Materialization failure here is a hard + // error: pass 1 already materialized both sides of every changed row. + if !changed.is_empty() { + let mut verdicts: FxHashMap<(Address, Address), bool> = + FxHashMap::default(); + let ripple_total = changed.len(); + let mut roots: usize = 0; + for (i, (_, name, ch)) in changed.iter_mut().enumerate() { + let key = (ch.old_addr.clone(), ch.new_addr.clone()); + let rippled = match verdicts.get(&key) { + Some(v) => *v, + None => { + let ca = materialize_const(a, &ch.old_addr, name, "first")?; + let cb = materialize_const(b, &ch.new_addr, name, "second")?; + let labels = classify_constants(a, &ca, b, &cb, Some(&addr_map))?; + let v = rippled_labels(&labels); + verdicts.insert(key, v); + v + }, + }; + ch.rippled = rippled; + if !rippled { + roots += 1; + } + let done = i + 1; + if done.is_multiple_of(RIPPLE_PROGRESS_STRIDE) && done < ripple_total { + on_progress(JoinProgress { + phase: DiffPhase::RippleClassify, + done, + total: ripple_total, + changed: roots, + }); + } + } + on_progress(JoinProgress { + phase: DiffPhase::RippleClassify, + done: ripple_total, + total: ripple_total, + changed: roots, + }); + } + for entry in b.named.iter() { + let name = entry.key(); + if a.named.get(name).is_none() { + added.push((name.pretty(), name.clone(), entry.value().addr.clone())); + } + } + sort_by_name(&mut added); + sort_by_name(&mut removed); + sort_by_name(&mut changed); + sort_by_name(&mut meta_only); + d.named_added = added.into_iter().map(|(s, _, addr)| (s, addr)).collect(); + d.named_removed = removed.into_iter().map(|(s, _, addr)| (s, addr)).collect(); + d.named_changed = changed.into_iter().map(|(_, _, c)| c).collect(); + d.named_meta_only = meta_only.into_iter().map(|(s, _, l)| (s, l)).collect(); + + // Consts / blobs set differences. + for entry in a.consts.iter() { + if !b.consts.contains_key(entry.key()) { + d.consts_only_a.push(entry.key().clone()); + } + } + for entry in b.consts.iter() { + if !a.consts.contains_key(entry.key()) { + d.consts_only_b.push(entry.key().clone()); + } + } + for entry in a.blobs.iter() { + if !b.blobs.contains_key(entry.key()) { + d.blobs_only_a.push(entry.key().clone()); + } + } + for entry in b.blobs.iter() { + if !a.blobs.contains_key(entry.key()) { + d.blobs_only_b.push(entry.key().clone()); + } + } + + // Comms join. + for entry in a.comms.iter() { + match b.comms.get(entry.key()) { + None => d.comms_removed.push(entry.key().clone()), + Some(other) => { + if other.value() != entry.value() { + d.comms_changed.push(entry.key().clone()); + } + }, + } + } + for entry in b.comms.iter() { + if a.comms.get(entry.key()).is_none() { + d.comms_added.push(entry.key().clone()); + } + } + + // Header: main + assumptions. + if a.main != b.main { + d.main_changed = Some((a.main.clone(), b.main.clone())); + } + for addr in &a.assumptions { + if !b.assumptions.contains(addr) { + d.assumptions_removed.push(addr.clone()); + } + } + for addr in &b.assumptions { + if !a.assumptions.contains(addr) { + d.assumptions_added.push(addr.clone()); + } + } + + // Hints, joined on constants present in both envs. + let shared = + |addr: &Address| a.consts.contains_key(addr) && b.consts.contains_key(addr); + for entry in a.anon_hints.iter() { + let (addr, ha) = (entry.key(), entry.value()); + if !shared(addr) { + continue; + } + let hb = b.anon_hints.get(addr).map(|r| *r); + if hb != Some(*ha) { + d.hints_changed.push(( + addr.clone(), + hint_label(Some(ha)), + hint_label(hb.as_ref()), + )); + } + } + for entry in b.anon_hints.iter() { + let (addr, hb) = (entry.key(), entry.value()); + if shared(addr) && !a.anon_hints.contains_key(addr) { + d.hints_changed.push(( + addr.clone(), + hint_label(None), + hint_label(Some(hb)), + )); + } + } + + d.consts_only_a.sort_unstable(); + d.consts_only_b.sort_unstable(); + d.blobs_only_a.sort_unstable(); + d.blobs_only_b.sort_unstable(); + d.comms_added.sort_unstable(); + d.comms_removed.sort_unstable(); + d.comms_changed.sort_unstable(); + d.assumptions_added.sort_unstable(); + d.assumptions_removed.sort_unstable(); + d.hints_changed.sort_unstable_by(|x, y| x.0.cmp(&y.0)); + + Ok(d) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::comm::Comm; + use crate::constant::{ + DefKind, ctor_proj_constant, defn_proj_constant, indc_proj_constant, + }; + use crate::metadata::{ConstantMeta, ConstantMetaInfo, ExprMeta}; + use ix_common::env::DefinitionSafety; + use std::sync::Arc; + + fn n(s: &str) -> Name { + Name::str(Name::anon(), s.to_string()) + } + + /// Store `c` at its true content address; returns the address. + fn store_canonical(env: &Env, c: &Constant) -> Address { + let (addr, _) = c.commit(); + env.store_const(addr.clone(), c.clone()); + addr + } + + fn mk_defn(lvls: u64, typ: Arc, value: Arc) -> Definition { + Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls, + typ, + value, + } + } + + fn defn_c(typ: Arc, value: Arc) -> Constant { + Constant::new(ConstantInfo::Defn(mk_defn(0, typ, value))) + } + + fn defn_ct( + typ: Arc, + value: Arc, + sharing: Vec>, + refs: Vec
, + univs: Vec>, + ) -> Constant { + Constant::with_tables( + ConstantInfo::Defn(mk_defn(0, typ, value)), + sharing, + refs, + univs, + ) + } + + /// Single-constant env: store `c` canonically and name it `name`. + fn env1(name: &str, c: &Constant) -> (Env, Address) { + let env = Env::new(); + let addr = store_canonical(&env, c); + env.register_name(n(name), Named::with_addr(addr.clone())); + (env, addr) + } + + fn labels(v: &[String]) -> Vec<&str> { + v.iter().map(String::as_str).collect() + } + + fn diff(a: &Env, b: &Env) -> EnvDiff { + diff_envs(a, b, false).expect("diff_envs failed") + } + + fn diff_meta(a: &Env, b: &Env) -> EnvDiff { + diff_envs(a, b, true).expect("diff_envs failed") + } + + #[test] + fn identical_envs_empty_diff() { + let mut env = Env::new(); + let c = defn_c(Expr::var(3), Expr::var(0)); + let addr = store_canonical(&env, &c); + env.register_name(n("Foo"), Named::with_addr(addr.clone())); + let blob = env.store_blob(vec![1, 2, 3]); + env.store_comm( + Address::hash(b"comm"), + Comm::new(Address::hash(b"s"), Address::hash(b"p")), + ); + env.main = Some(addr.clone()); + env.assumptions.insert(blob); + env.anon_hints.insert(addr, ReducibilityHints::Regular(2)); + let copy = env.clone(); + let d = diff(&env, ©); + assert!(d.is_empty(), "anon self-diff should be empty: {d:?}"); + assert_eq!(d.stats_a, d.stats_b); + assert_eq!(d.stats_a.consts, 1); + assert!(diff_meta(&env, ©).is_empty(), "meta self-diff not empty"); + } + + #[test] + fn added_and_removed_name() { + let (a, _) = env1("Base", &defn_c(Expr::var(3), Expr::var(0))); + let b = a.clone(); + let extra = defn_c(Expr::var(4), Expr::var(1)); + let extra_addr = store_canonical(&b, &extra); + b.register_name(n("Extra"), Named::with_addr(extra_addr.clone())); + + let d = diff(&a, &b); + assert_eq!(d.named_added, vec![("Extra".to_string(), extra_addr.clone())]); + assert!(d.named_removed.is_empty() && d.named_changed.is_empty()); + assert_eq!(d.consts_only_b, vec![extra_addr.clone()]); + assert!(d.consts_only_a.is_empty()); + + let d = diff(&b, &a); + assert_eq!( + d.named_removed, + vec![("Extra".to_string(), extra_addr.clone())] + ); + assert_eq!(d.consts_only_a, vec![extra_addr]); + } + + #[test] + fn defn_value_only_changed() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(1))); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 1); + let c = &d.named_changed[0]; + assert_eq!(c.name, "Foo"); + assert_eq!((c.old_kind, c.new_kind), ("defn", "defn")); + assert_eq!(labels(&c.fields), ["value"]); + assert!(c.meta_fields.is_empty()); + } + + #[test] + fn defn_type_only_changed() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(4), Expr::var(0))); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + } + + #[test] + fn defn_scalars_changed() { + let ca = Constant::new(ConstantInfo::Defn(Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls: 0, + typ: Expr::var(3), + value: Expr::var(0), + })); + let cb = Constant::new(ConstantInfo::Defn(Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Partial, + lvls: 1, + typ: Expr::var(3), + value: Expr::var(0), + })); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["safety", "lvls"]); + } + + /// Identical expr bytes over different refs tables are DIFFERENT. + #[test] + fn refs_shift_same_bytes_is_different() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let mk = |r: Address| { + defn_ct(Expr::var(9), Expr::reference(0, vec![]), vec![], vec![r], vec![]) + }; + let (a, _) = env1("Foo", &mk(x)); + let (b, _) = env1("Foo", &mk(y)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["value"]); + } + + /// Different indices resolving to the same addresses with the same + /// structure are EQUAL — a real type change is the only field. + #[test] + fn refs_permuted_same_resolution_is_equal() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(2), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y.clone(), x.clone()], + vec![], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + + // Pure permutation: no semantic field difference at all. + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y, x], + vec![], + ); + let (a, addr_a) = env1("Foo", &ca); + let (b, addr_b) = env1("Foo", &cb); + assert_ne!(addr_a, addr_b, "permuted tables must change the address"); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + } + + /// Sharing-table indirection vs inlined subterms: resolved-equal. + #[test] + fn share_vs_inline_equal() { + let t = Expr::all(Expr::var(7), Expr::var(0)); + let ca = defn_ct( + Expr::var(1), + Expr::app(Arc::new(Expr::Share(0)), Arc::new(Expr::Share(0))), + vec![t.clone()], + vec![], + vec![], + ); + let cb = + defn_ct(Expr::var(1), Expr::app(t.clone(), t), vec![], vec![], vec![]); + let (a, addr_a) = env1("Foo", &ca); + let (b, addr_b) = env1("Foo", &cb); + assert_ne!(addr_a, addr_b); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + } + + /// Univ-table shifts with identical resolution are not a type change; + /// changed univ content behind the same index is. + #[test] + fn univ_table_shift_equal_content_change_not() { + let ca = + defn_ct(Expr::sort(0), Expr::var(0), vec![], vec![], vec![Univ::zero()]); + let cb = defn_ct( + Expr::sort(1), + Expr::var(0), + vec![], + vec![], + vec![Univ::succ(Univ::zero()), Univ::zero()], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + + let ca = + defn_ct(Expr::sort(0), Expr::var(0), vec![], vec![], vec![Univ::zero()]); + let cb = defn_ct( + Expr::sort(0), + Expr::var(0), + vec![], + vec![], + vec![Univ::succ(Univ::zero())], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + } + + /// A field-compare failure must not leave stale memo entries that + /// mask a later field's difference (type and value share Arcs here). + #[test] + fn memo_cleared_after_unequal_field() { + let e_a = Expr::app(Expr::var(1), Expr::var(5)); + let e_b = Expr::app(Expr::var(1), Expr::var(6)); + let ca = defn_c(e_a.clone(), e_a); + let cb = defn_c(e_b.clone(), e_b); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type", "value"]); + } + + #[test] + fn metadata_only_change() { + let c = defn_c(Expr::var(3), Expr::var(0)); + let (a, addr) = env1("Foo", &c); + let b = Env::new(); + store_canonical(&b, &c); + let meta = ConstantMeta { + meta_refs: vec![Address::hash(b"extra-ref")], + ..ConstantMeta::default() + }; + b.register_name(n("Foo"), Named::new(addr.clone(), meta)); + + let d = diff(&a, &b); + assert!(d.is_empty(), "anon mode must ignore metadata: {d:?}"); + let d = diff_meta(&a, &b); + assert_eq!(d.named_meta_only.len(), 1); + assert_eq!(d.named_meta_only[0].0, "Foo"); + assert_eq!(labels(&d.named_meta_only[0].1), ["meta.refs"]); + assert!(d.named_changed.is_empty()); + + // original added + info kind change carry their own labels. + let b2 = Env::new(); + store_canonical(&b2, &c); + let info = ConstantMetaInfo::Def { + name: Address::hash(b"nm"), + lvls: vec![], + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }; + let mut named2 = Named::new(addr.clone(), ConstantMeta::new(info)); + named2.set_original(addr.clone(), ConstantMeta::default()); + b2.register_name(n("Foo"), named2); + let d = diff_meta(&a, &b2); + assert_eq!( + labels(&d.named_meta_only[0].1), + ["meta.info(empty→def)", "original.added"] + ); + } + + #[test] + fn kind_change() { + let ca = Constant::new(ConstantInfo::Axio(crate::constant::Axiom { + is_unsafe: false, + lvls: 0, + typ: Expr::var(3), + })); + let cb = defn_c(Expr::var(3), Expr::var(0)); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + let c = &d.named_changed[0]; + assert_eq!((c.old_kind, c.new_kind), ("axio", "defn")); + assert_eq!(labels(&c.fields), ["kind"]); + } + + #[test] + fn main_assumptions_comms_hints() { + let shared = defn_c(Expr::var(3), Expr::var(0)); + let p = Address::hash(b"p"); + let q = Address::hash(b"q"); + let r = Address::hash(b"r"); + let c1 = Address::hash(b"comm1"); + let c2 = Address::hash(b"comm2"); + + let mut a = Env::new(); + let addr = store_canonical(&a, &shared); + a.main = Some(addr.clone()); + a.assumptions.insert(p.clone()); + a.assumptions.insert(q.clone()); + a.store_comm( + c1.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay1")), + ); + a.anon_hints.insert(addr.clone(), ReducibilityHints::Regular(1)); + + let mut b = Env::new(); + store_canonical(&b, &shared); + b.main = Some(p.clone()); + b.assumptions.insert(q); + b.assumptions.insert(r.clone()); + b.store_comm( + c1.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay2")), + ); + b.store_comm( + c2.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay3")), + ); + b.anon_hints.insert(addr.clone(), ReducibilityHints::Regular(2)); + + let d = diff(&a, &b); + assert_eq!(d.main_changed, Some((Some(addr.clone()), Some(p.clone())))); + assert_eq!(d.assumptions_added, vec![r]); + assert_eq!(d.assumptions_removed, vec![p]); + assert_eq!(d.comms_changed, vec![c1]); + assert_eq!(d.comms_added, vec![c2]); + assert!(d.comms_removed.is_empty()); + assert_eq!( + d.hints_changed, + vec![(addr, "regular(1)".to_string(), "regular(2)".to_string())] + ); + } + + /// One-level block descent: the changed member gets `block.*` detail, + /// the untouched sibling gets `block-siblings`. + #[test] + fn projection_block_descent() { + let f = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(0))); + let g_old = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(1))); + let g_new = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(2))); + let block_a = Constant::new(ConstantInfo::Muts(vec![f.clone(), g_old])); + let block_b = Constant::new(ConstantInfo::Muts(vec![f, g_new])); + + let a = Env::new(); + let block_a_addr = store_canonical(&a, &block_a); + let fa = store_canonical(&a, &defn_proj_constant(0, block_a_addr.clone())); + let ga = store_canonical(&a, &defn_proj_constant(1, block_a_addr)); + a.register_name(n("M.f"), Named::with_addr(fa)); + a.register_name(n("M.g"), Named::with_addr(ga)); + + let b = Env::new(); + let block_b_addr = store_canonical(&b, &block_b); + let fb = store_canonical(&b, &defn_proj_constant(0, block_b_addr.clone())); + let gb = store_canonical(&b, &defn_proj_constant(1, block_b_addr)); + b.register_name(n("M.f"), Named::with_addr(fb)); + b.register_name(n("M.g"), Named::with_addr(gb)); + + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 2); + let mf = &d.named_changed[0]; + let mg = &d.named_changed[1]; + assert_eq!((mf.name.as_str(), mg.name.as_str()), ("M.f", "M.g")); + assert_eq!((mf.old_kind, mf.new_kind), ("dprj", "dprj")); + assert_eq!(labels(&mf.fields), ["block-siblings"]); + assert_eq!(labels(&mg.fields), ["block.value"]); + } + + /// Anon-mode diffs must be identical whether the envs came from the + /// full reader or the lazy-index path (`ix diff`'s memory-lean route). + #[test] + fn lazy_index_env_matches_full_reader_in_anon_mode() { + let build = |value: Arc, hint: u32, with_extra: bool| { + let mut env = Env::new(); + let register = |env: &Env, label: &str, addr: Address| { + let name = n(label); + env + .names + .insert(Address::from_blake3_hash(*name.get_hash()), name.clone()); + env.register_name(name, Named::with_addr(addr)); + }; + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + register(&env, "Foo", addr.clone()); + // Hint lives on a constant PRESENT IN BOTH envs — the hints diff + // joins on shared consts only. + let stable = defn_c(Expr::var(7), Expr::var(0)); + let stable_addr = store_canonical(&env, &stable); + register(&env, "Stable", stable_addr.clone()); + env.anon_hints.insert(stable_addr, ReducibilityHints::Regular(hint)); + env.store_blob(vec![1, 2, 3]); + env.store_comm( + Address::hash(b"comm"), + Comm::new(Address::hash(b"s"), Address::hash(b"p")), + ); + env.main = Some(addr); + env.assumptions.insert(Address::hash(b"assumed")); + if with_extra { + let extra = defn_c(Expr::var(5), Expr::var(2)); + let extra_addr = store_canonical(&env, &extra); + register(&env, "Extra", extra_addr); + } + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let bytes_a = build(Expr::var(0), 1, false); + let bytes_b = build(Expr::var(1), 2, true); + + let full = |bytes: &[u8]| { + let mut cursor = bytes; + Env::get(&mut cursor).expect("full read failed") + }; + let lazy = |bytes: &[u8]| { + let index = Env::parse_lazy_index(bytes).expect("lazy index failed"); + Env::from_lazy_index(&index, bytes).expect("from_lazy_index failed") + }; + + let via_full = diff(&full(&bytes_a), &full(&bytes_b)); + let via_lazy = diff(&lazy(&bytes_a), &lazy(&bytes_b)); + assert_eq!(via_full, via_lazy); + // Sanity: the pair genuinely exercises every section. + assert_eq!(via_full.named_changed.len(), 1); + assert_eq!(via_full.named_added.len(), 1); + assert_eq!(via_full.hints_changed.len(), 1); + // Lazy self-diff is empty. + assert!(diff(&lazy(&bytes_a), &lazy(&bytes_a)).is_empty()); + } + + #[test] + fn join_progress_final_event_totals() { + let (a, _) = env1("Base", &defn_c(Expr::var(3), Expr::var(0))); + let b = a.clone(); + let extra = defn_c(Expr::var(4), Expr::var(1)); + let extra_addr = store_canonical(&b, &extra); + b.register_name(n("Extra"), Named::with_addr(extra_addr)); + // Change Base's constant so exactly one join entry classifies. + let changed = defn_c(Expr::var(3), Expr::var(9)); + let changed_addr = store_canonical(&b, &changed); + b.register_name(n("Base"), Named::with_addr(changed_addr)); + + let mut events: Vec = Vec::new(); + let d = diff_envs_with(&a, &b, false, &mut |p| events.push(p)) + .expect("diff_envs_with failed"); + // `done` resets between phases — assert each phase separately. + let join: Vec<_> = + events.iter().filter(|p| p.phase == DiffPhase::NamedJoin).collect(); + let ripple: Vec<_> = + events.iter().filter(|p| p.phase == DiffPhase::RippleClassify).collect(); + let last_join = join.last().expect("no join events"); + assert_eq!(last_join.done, last_join.total); + assert_eq!(last_join.total, a.named.len()); + assert_eq!(last_join.changed, d.named_changed.len()); + assert_eq!(d.named_changed.len(), 1); + let last_ripple = ripple.last().expect("no ripple events"); + assert_eq!(last_ripple.done, last_ripple.total); + assert_eq!(last_ripple.total, d.named_changed.len()); + assert_eq!( + last_ripple.changed, + d.named_changed.iter().filter(|c| !c.rippled).count() + ); + for phase_events in [&join, &ripple] { + assert!( + phase_events.windows(2).all(|w| w[0].done <= w[1].done), + "per-phase progress must be monotone" + ); + } + // All ripple events come after the join completes. + let first_ripple = events + .iter() + .position(|p| p.phase == DiffPhase::RippleClassify) + .expect("no ripple events"); + let last_join_pos = events + .iter() + .rposition(|p| p.phase == DiffPhase::NamedJoin) + .expect("no join events"); + assert!(last_join_pos < first_ripple, "phases must not interleave"); + } + + #[test] + fn recursor_rules_changed() { + let mk = |rhs: Arc, rules_extra: bool| { + let mut rules = vec![crate::constant::RecursorRule { fields: 2, rhs }]; + if rules_extra { + rules + .push(crate::constant::RecursorRule { fields: 0, rhs: Expr::var(0) }); + } + Constant::new(ConstantInfo::Recr(Recursor { + k: false, + is_unsafe: false, + lvls: 1, + params: 1, + indices: 0, + motives: 1, + minors: 1, + typ: Expr::var(9), + rules, + })) + }; + let (a, _) = env1("R", &mk(Expr::var(4), false)); + let (b, _) = env1("R", &mk(Expr::var(5), false)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["rules[0].rhs"]); + + let (a, _) = env1("R", &mk(Expr::var(4), false)); + let (b, _) = env1("R", &mk(Expr::var(4), true)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["rules.len"]); + } + + // ========================================================================== + // Ripple root-causing (pass 2) + // ========================================================================== + + /// A defn whose value is a bare `Ref` into `refs[0] = r`. + fn defn_ref(typ_var: u64, r: Address) -> Constant { + defn_ct( + Expr::var(typ_var), + Expr::reference(0, vec![]), + vec![], + vec![r], + vec![], + ) + } + + fn changed_by_name<'d>(d: &'d EnvDiff, name: &str) -> &'d NamedChange { + d.named_changed + .iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("no changed row named {name}: {d:?}")) + } + + /// Editing a leaf re-addresses the chain above it; only the leaf is a + /// root. Proves the single-level map composes across the DAG through + /// per-row verdicts. + #[test] + fn ripple_two_hop_chain() { + let build = |leaf_value: Arc| { + let env = Env::new(); + let leaf = defn_c(Expr::var(3), leaf_value); + let leaf_addr = store_canonical(&env, &leaf); + let mid = defn_ref(4, leaf_addr.clone()); + let mid_addr = store_canonical(&env, &mid); + let top = defn_ref(5, mid_addr.clone()); + let top_addr = store_canonical(&env, &top); + env.register_name(n("Leaf"), Named::with_addr(leaf_addr)); + env.register_name(n("Mid"), Named::with_addr(mid_addr)); + env.register_name(n("Top"), Named::with_addr(top_addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 3); + let leaf = changed_by_name(&d, "Leaf"); + assert!(!leaf.rippled, "the edited leaf is the root"); + assert_eq!(labels(&leaf.fields), ["value"]); + for name in ["Mid", "Top"] { + let c = changed_by_name(&d, name); + assert!(c.rippled, "{name} must be rippled: {c:?}"); + // Strict fields are untouched by the verdict. + assert_eq!(labels(&c.fields), ["value"]); + } + } + + /// Blob addresses never enter the quotient map: a literal edit is the + /// intrinsic change site. + #[test] + fn ripple_literal_change_is_root() { + let build = |blob: Vec| { + let env = Env::new(); + let blob_addr = env.store_blob(blob); + let c = + defn_ct(Expr::var(3), Expr::nat(0), vec![], vec![blob_addr], vec![]); + let addr = store_canonical(&env, &c); + env.register_name(n("Lit"), Named::with_addr(addr)); + env + }; + let a = build(vec![42]); + let b = build(vec![43]); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Lit"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["value"]); + } + + /// Block-internal ctor edit: the edited ctor's projection is the root + /// (block descent runs under the quotient too); the untouched sibling + /// ctor and any external dependent are rippled. + #[test] + fn ripple_block_ctor_edit_projections() { + let build = |c1_typ: Arc| { + let env = Env::new(); + let indc = Inductive { + is_unsafe: false, + lvls: 0, + params: 0, + indices: 0, + typ: Expr::var(3), + ctors: vec![ + Constructor { + is_unsafe: false, + lvls: 0, + cidx: 0, + params: 0, + fields: 0, + typ: Expr::var(1), + }, + Constructor { + is_unsafe: false, + lvls: 0, + cidx: 1, + params: 0, + fields: 0, + typ: c1_typ, + }, + ], + }; + let block = Constant::new(ConstantInfo::Muts(vec![MutConst::Indc(indc)])); + let block_addr = store_canonical(&env, &block); + let iprj = + store_canonical(&env, &indc_proj_constant(0, block_addr.clone())); + let c0 = + store_canonical(&env, &ctor_proj_constant(0, 0, block_addr.clone())); + let c1 = store_canonical(&env, &ctor_proj_constant(0, 1, block_addr)); + env.register_name(n("I"), Named::with_addr(iprj.clone())); + env.register_name(n("I.c0"), Named::with_addr(c0)); + env.register_name(n("I.c1"), Named::with_addr(c1)); + // External dependent referencing the inductive's projection. + let dep = defn_ref(9, iprj); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("UsesI"), Named::with_addr(dep_addr)); + env + }; + let a = build(Expr::var(2)); + let b = build(Expr::var(9)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 4); + + let edited = changed_by_name(&d, "I.c1"); + assert!(!edited.rippled, "edited ctor's projection is the root"); + assert_eq!(labels(&edited.fields), ["block.ctor.type"]); + + let indc_row = changed_by_name(&d, "I"); + assert!(!indc_row.rippled, "the inductive sees its ctor's edit"); + assert_eq!(labels(&indc_row.fields), ["block.ctors[1].type"]); + + let sibling = changed_by_name(&d, "I.c0"); + assert!(sibling.rippled, "untouched sibling ctor is rippled"); + assert_eq!(labels(&sibling.fields), ["block-siblings"]); + + let dep = changed_by_name(&d, "UsesI"); + assert!(dep.rippled, "external dependent of the block is rippled"); + assert_eq!(labels(&dep.fields), ["value"]); + } + + /// The existing block-descent fixture, now with verdicts: the member + /// whose value changed is the root; its sibling projection rides. + #[test] + fn ripple_defn_block_sibling() { + let f = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(0))); + let g_old = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(1))); + let g_new = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(2))); + let block_a = Constant::new(ConstantInfo::Muts(vec![f.clone(), g_old])); + let block_b = Constant::new(ConstantInfo::Muts(vec![f, g_new])); + + let a = Env::new(); + let block_a_addr = store_canonical(&a, &block_a); + let fa = store_canonical(&a, &defn_proj_constant(0, block_a_addr.clone())); + let ga = store_canonical(&a, &defn_proj_constant(1, block_a_addr)); + a.register_name(n("M.f"), Named::with_addr(fa)); + a.register_name(n("M.g"), Named::with_addr(ga)); + + let b = Env::new(); + let block_b_addr = store_canonical(&b, &block_b); + let fb = store_canonical(&b, &defn_proj_constant(0, block_b_addr.clone())); + let gb = store_canonical(&b, &defn_proj_constant(1, block_b_addr)); + b.register_name(n("M.f"), Named::with_addr(fb)); + b.register_name(n("M.g"), Named::with_addr(gb)); + + let d = diff(&a, &b); + assert!(changed_by_name(&d, "M.f").rippled); + assert!(!changed_by_name(&d, "M.g").rippled); + } + + /// Scalar changes are quotient-invariant → always roots. + #[test] + fn ripple_lvls_change_is_root() { + let (a, _) = env1( + "Foo", + &Constant::new(ConstantInfo::Defn(mk_defn( + 0, + Expr::var(3), + Expr::var(0), + ))), + ); + let (b, _) = env1( + "Foo", + &Constant::new(ConstantInfo::Defn(mk_defn( + 1, + Expr::var(3), + Expr::var(0), + ))), + ); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["lvls"]); + } + + /// Pure representation churn (strict `["encoding"]`) has no intrinsic + /// difference under the quotient either → rippled. + #[test] + fn ripple_pure_encoding_is_rippled() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y, x], + vec![], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert_eq!(labels(&c.fields), ["encoding"]); + assert!(c.rippled); + } + + /// Kind-change rows enter the map too: their dependents are rippled. + #[test] + fn ripple_kind_change_root_and_maps() { + let build = |axio: bool| { + let env = Env::new(); + let dc: Constant = if axio { + Constant::new(ConstantInfo::Axio(crate::constant::Axiom { + is_unsafe: false, + lvls: 0, + typ: Expr::var(3), + })) + } else { + defn_c(Expr::var(3), Expr::var(0)) + }; + let d_addr = store_canonical(&env, &dc); + let dep = defn_ref(9, d_addr.clone()); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("D"), Named::with_addr(d_addr)); + env.register_name(n("UsesD"), Named::with_addr(dep_addr)); + env + }; + let a = build(true); + let b = build(false); + let d = diff(&a, &b); + let kd = changed_by_name(&d, "D"); + assert!(!kd.rippled); + assert_eq!(labels(&kd.fields), ["kind"]); + let dep = changed_by_name(&d, "UsesD"); + assert!(dep.rippled, "kind rows must still enter the quotient map"); + } + + /// Two names sharing one old address, diverging in the new env: the + /// set-valued map explains both dependents. + #[test] + fn ripple_name_split_conflict() { + let a = Env::new(); + let x = defn_c(Expr::var(3), Expr::var(0)); + let x_addr = store_canonical(&a, &x); + a.register_name(n("P"), Named::with_addr(x_addr.clone())); + a.register_name(n("Q"), Named::with_addr(x_addr.clone())); + let dp = defn_ref(7, x_addr.clone()); + let dp_addr = store_canonical(&a, &dp); + a.register_name(n("DP"), Named::with_addr(dp_addr)); + let dq = defn_ref(8, x_addr); + let dq_addr = store_canonical(&a, &dq); + a.register_name(n("DQ"), Named::with_addr(dq_addr)); + + let b = Env::new(); + let y1 = defn_c(Expr::var(3), Expr::var(1)); + let y1_addr = store_canonical(&b, &y1); + let y2 = defn_c(Expr::var(3), Expr::var(2)); + let y2_addr = store_canonical(&b, &y2); + b.register_name(n("P"), Named::with_addr(y1_addr.clone())); + b.register_name(n("Q"), Named::with_addr(y2_addr.clone())); + let dp2 = defn_ref(7, y1_addr); + let dp2_addr = store_canonical(&b, &dp2); + b.register_name(n("DP"), Named::with_addr(dp2_addr)); + let dq2 = defn_ref(8, y2_addr); + let dq2_addr = store_canonical(&b, &dq2); + b.register_name(n("DQ"), Named::with_addr(dq2_addr)); + + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "P").rippled); + assert!(!changed_by_name(&d, "Q").rippled); + assert!(changed_by_name(&d, "DP").rippled); + assert!(changed_by_name(&d, "DQ").rippled); + } + + /// Ref univ arguments: same args over a mapped target → rippled; + /// changed arg arity (induced re-elaboration) → root. + #[test] + fn ripple_ref_univ_args() { + let build = |leaf_value: Arc, dep_univs: bool| { + let env = Env::new(); + let leaf = + Constant::new(ConstantInfo::Defn(mk_defn(1, Expr::var(3), leaf_value))); + let leaf_addr = store_canonical(&env, &leaf); + let dep = if dep_univs { + defn_ct( + Expr::var(9), + Expr::reference(0, vec![0, 1]), + vec![], + vec![leaf_addr.clone()], + vec![Univ::zero(), Univ::succ(Univ::zero())], + ) + } else { + defn_ct( + Expr::var(9), + Expr::reference(0, vec![0]), + vec![], + vec![leaf_addr.clone()], + vec![Univ::zero()], + ) + }; + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("Leaf"), Named::with_addr(leaf_addr)); + env.register_name(n("Dep"), Named::with_addr(dep_addr)); + env + }; + // Same univ args over a mapped target: rippled. + let a = build(Expr::var(0), false); + let b = build(Expr::var(1), false); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "Leaf").rippled); + assert!(changed_by_name(&d, "Dep").rippled); + + // Univ-argument arity changed at the use site: intrinsic (induced + // re-elaboration) → root, even though the target maps. + let a = build(Expr::var(0), false); + let b = build(Expr::var(1), true); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "Leaf").rippled); + let dep = changed_by_name(&d, "Dep"); + assert!(!dep.rippled); + assert_eq!(labels(&dep.fields), ["value"]); + } + + /// A dependency renamed (removed+added) never enters the map: its + /// dependents verdict root — fail-safe over-report. + #[test] + fn ripple_renamed_dep_is_root() { + let a = Env::new(); + let old = defn_c(Expr::var(3), Expr::var(0)); + let old_addr = store_canonical(&a, &old); + a.register_name(n("OldDep"), Named::with_addr(old_addr.clone())); + let dep = defn_ref(9, old_addr); + let dep_addr = store_canonical(&a, &dep); + a.register_name(n("User"), Named::with_addr(dep_addr)); + + let b = Env::new(); + let new = defn_c(Expr::var(3), Expr::var(1)); + let new_addr = store_canonical(&b, &new); + b.register_name(n("NewDep"), Named::with_addr(new_addr.clone())); + let dep2 = defn_ref(9, new_addr); + let dep2_addr = store_canonical(&b, &dep2); + b.register_name(n("User"), Named::with_addr(dep2_addr)); + + let d = diff(&a, &b); + assert_eq!(d.named_removed.len(), 1); + assert_eq!(d.named_added.len(), 1); + let user = changed_by_name(&d, "User"); + assert!(!user.rippled, "renamed dep is not in the map → root"); + } + + /// The third address site: `Expr::Prj` type targets quotient too. + #[test] + fn ripple_expr_prj_type_mapped() { + let build = |t_value: Arc| { + let env = Env::new(); + let t = defn_c(Expr::var(3), t_value); + let t_addr = store_canonical(&env, &t); + let dep = defn_ct( + Expr::var(9), + Expr::prj(0, 0, Expr::var(0)), + vec![], + vec![t_addr.clone()], + vec![], + ); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("T"), Named::with_addr(t_addr)); + env.register_name(n("UsesT"), Named::with_addr(dep_addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "T").rippled); + assert!(changed_by_name(&d, "UsesT").rippled); + } + + /// Changed projection coordinates target a different member: root. + #[test] + fn ripple_cprj_cidx_change_is_root() { + let block_a = Constant::new(ConstantInfo::Muts(vec![MutConst::Defn( + mk_defn(0, Expr::var(3), Expr::var(0)), + )])); + let block_b = Constant::new(ConstantInfo::Muts(vec![MutConst::Defn( + mk_defn(0, Expr::var(3), Expr::var(1)), + )])); + let a = Env::new(); + let ba = store_canonical(&a, &block_a); + let pa = store_canonical(&a, &ctor_proj_constant(0, 0, ba)); + a.register_name(n("C"), Named::with_addr(pa)); + let b = Env::new(); + let bb = store_canonical(&b, &block_b); + let pb = store_canonical(&b, &ctor_proj_constant(0, 1, bb)); + b.register_name(n("C"), Named::with_addr(pb)); + let d = diff(&a, &b); + let c = changed_by_name(&d, "C"); + assert_eq!(labels(&c.fields), ["cidx", "block"]); + assert!(!c.rippled); + } + + /// Alias rows spanning one (old, new) pair share one cached verdict. + #[test] + fn ripple_alias_rows_consistent() { + let build = |value: Arc| { + let env = Env::new(); + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + env.register_name(n("P"), Named::with_addr(addr.clone())); + env.register_name(n("Q"), Named::with_addr(addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 2); + let p = changed_by_name(&d, "P"); + let q = changed_by_name(&d, "Q"); + assert_eq!(p.rippled, q.rippled); + assert!(!p.rippled); + assert_eq!( + (p.old_addr.clone(), p.new_addr.clone()), + (q.old_addr.clone(), q.new_addr.clone()) + ); + } + + /// A lone changed leaf with no dependents: verdict root, and strict + /// fields identical to the pre-ripple behavior. + #[test] + fn ripple_single_root_no_map_hits() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(1))); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["value"]); + } + + // ========================================================================== + // Streaming §5 meta sweep (diff_env_bytes) + // ========================================================================== + + /// Register `name`'s component in `env.names`; returns its address. + fn register_component(env: &Env, name: &Name) -> Address { + let addr = Address::from_blake3_hash(*name.get_hash()); + env.names.insert(addr.clone(), name.clone()); + addr + } + + /// `variant` lands in `type_root`, so two calls with different + /// variants produce metas that differ while the constant (and its + /// address) stays identical — a metadata-only difference. + fn def_meta(name_addr: Address, variant: u32) -> ConstantMeta { + ConstantMeta::new(ConstantMetaInfo::Def { + name: name_addr, + lvls: vec![], + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: u64::from(variant), + value_root: 0, + }) + } + + fn full_read(bytes: &[u8]) -> Env { + let mut cur = bytes; + Env::get(&mut cur).expect("full read failed") + } + + /// The streaming §5 sweep must reproduce the full reader's meta-mode + /// report exactly: `named_meta_only`, `meta_fields` on changed rows, + /// and everything structural. + #[test] + fn meta_sweep_matches_full_reader() { + let build = |foo_value: Arc, + foo_variant: u32, + monly_variant: u32, + foo_original: bool| + -> Vec { + let env = Env::new(); + let foo = n("Foo"); + let foo_comp = register_component(&env, &foo); + let c = defn_c(Expr::var(3), foo_value); + let addr = store_canonical(&env, &c); + let mut foo_named = + Named::new(addr.clone(), def_meta(foo_comp, foo_variant)); + if foo_original { + foo_named.set_original(addr, ConstantMeta::default()); + } + env.register_name(foo.clone(), foo_named); + // Same const both sides; only metadata differs → named_meta_only. + let monly = n("MetaOnly"); + let monly_comp = register_component(&env, &monly); + let stable = defn_c(Expr::var(7), Expr::var(0)); + let stable_addr = store_canonical(&env, &stable); + env.register_name( + monly.clone(), + Named::new(stable_addr, def_meta(monly_comp, monly_variant)), + ); + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let ba = build(Expr::var(0), 1, 3, false); + let bb = build(Expr::var(1), 2, 4, true); + + let via_full = + diff_envs(&full_read(&ba), &full_read(&bb), true).expect("full diff"); + let mut events: Vec = Vec::new(); + let via_sweep = diff_env_bytes(&ba, &bb, true, &mut |p| events.push(p)) + .expect("sweep diff"); + assert_eq!(via_full, via_sweep); + // The pair genuinely exercises both meta categories. + assert_eq!(via_full.named_meta_only.len(), 1); + assert_eq!(via_full.named_meta_only[0].0, "MetaOnly"); + let foo_row = + via_full.named_changed.iter().find(|c| c.name == "Foo").unwrap(); + assert!(!foo_row.meta_fields.is_empty()); + // The sweep phase fired and completed before the join started. + assert_eq!(events.first().map(|p| p.phase), Some(DiffPhase::MetaSweep)); + let last_sweep = + events.iter().rfind(|p| p.phase == DiffPhase::MetaSweep).unwrap(); + assert_eq!(last_sweep.done, last_sweep.total); + + // Anon parity through the bytes path too. + let anon_full = + diff_envs(&full_read(&ba), &full_read(&bb), false).expect("anon full"); + let anon_sweep = + diff_env_bytes(&ba, &bb, false, &mut |_| {}).expect("anon sweep"); + assert_eq!(anon_full, anon_sweep); + // Self-diff via the bytes path is empty in both modes. + assert!(diff_env_bytes(&ba, &ba, true, &mut |_| {}).unwrap().is_empty()); + assert!(diff_env_bytes(&ba, &ba, false, &mut |_| {}).unwrap().is_empty()); + } + + /// Metadata name references are file-relative §4 indices, so raw §5 + /// windows differ across files whose name tables differ — the sweep + /// must still see logically identical metadata as equal (it compares + /// parsed, Address-valued values, never bytes). + #[test] + fn meta_sweep_ignores_name_index_shift() { + let foo = n("Foo"); + let foo_comp = Address::from_blake3_hash(*foo.get_hash()); + // A name whose component address sorts BEFORE Foo's: its presence + // in env B shifts Foo's §4 index, re-encoding Foo's (identical) + // metadata over different indices. + let extra = (0..) + .map(|i| n(&format!("X{i}"))) + .find(|nm| Address::from_blake3_hash(*nm.get_hash()) < foo_comp) + .expect("some candidate component sorts before Foo"); + let build = |with_extra: bool| -> Vec { + let env = Env::new(); + register_component(&env, &foo); + let c = defn_c(Expr::var(3), Expr::var(0)); + let addr = store_canonical(&env, &c); + env.register_name( + foo.clone(), + Named::new(addr, def_meta(foo_comp.clone(), 1)), + ); + if with_extra { + register_component(&env, &extra); + let c2 = defn_c(Expr::var(8), Expr::var(2)); + let a2 = store_canonical(&env, &c2); + env.register_name(extra.clone(), Named::with_addr(a2)); + } + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let ba = build(false); + let bb = build(true); + let d = diff_env_bytes(&ba, &bb, true, &mut |_| {}).expect("sweep diff"); + assert_eq!(d.named_added.len(), 1); + assert!( + d.named_meta_only.is_empty(), + "index shift misread as a metadata diff: {d:?}" + ); + assert!(d.named_changed.is_empty()); + // And the full reader agrees. + assert_eq!( + d, + diff_envs(&full_read(&ba), &full_read(&bb), true).expect("full diff") + ); + } + + /// Mmap-backed lazy sides produce the same report as heap-backed + /// ones (the `rs_diff_env_files` path). + #[test] + fn mmap_lazy_side_matches_heap() { + let build = |value: Arc| -> Vec { + let env = Env::new(); + let foo = n("Foo"); + register_component(&env, &foo); + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + env.register_name(foo, Named::with_addr(addr)); + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let (ba, bb) = (build(Expr::var(0)), build(Expr::var(1))); + + let path = std::env::temp_dir() + .join(format!("ixon-diff-mmap-test-{}.ixe", std::process::id())); + std::fs::write(&path, &ba).expect("write temp"); + let file = std::fs::File::open(&path).expect("open temp"); + let mmap = + Arc::new(unsafe { memmap2::Mmap::map(&file) }.expect("mmap temp")); + let ia = Env::parse_lazy_index(&mmap[..]).expect("lazy index (mmap)"); + let ea = Env::from_lazy_index_mmap(&ia, &mmap).expect("from mmap"); + + let ib = Env::parse_lazy_index(&bb).expect("lazy index (heap)"); + let eb = Env::from_lazy_index(&ib, &bb).expect("from heap"); + + let via_mmap = diff_envs_lazy( + LazySide { env: &ea, index: &ia, data: &mmap[..] }, + LazySide { env: &eb, index: &ib, data: &bb }, + true, + &mut |_| {}, + ) + .expect("mmap-side diff"); + let via_heap = + diff_env_bytes(&ba, &bb, true, &mut |_| {}).expect("heap diff"); + assert_eq!(via_mmap, via_heap); + assert_eq!(via_heap.named_changed.len(), 1); + std::fs::remove_file(&path).ok(); + } +} diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index 3a5f46f58..a4bb3a1b8 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -1,6 +1,6 @@ //! Environment for storing Ixon data. -use rustc_hash::{FxHashMap, FxHashSet}; +use rustc_hash::FxHashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -8,7 +8,10 @@ use ix_common::address::Address; use ix_common::env::{Name, ReducibilityHints}; use super::comm::Comm; -use super::constant::Constant; +use super::constant::{ + Constant, ConstantInfo, MutConst, ctor_proj_address, defn_proj_address, + indc_proj_address, recr_proj_address, +}; use super::lazy::LazyConstant; use super::map::IxonMap; use super::metadata::{ConstantMeta, ConstantMetaInfo}; @@ -162,15 +165,14 @@ pub struct LazyConstSlice { pub len: usize, } -/// One named entry in a [`LazyIndex`]: just the `name → addr` mapping plus the -/// per-`Defn` reducibility hint (the only metadata the typecheck circuit -/// consumes). The heavy `ExprMetaArena` is parsed (to advance the cursor and -/// handle every metadata variant, e.g. `CallSite`) but discarded. +/// One named entry in a [`LazyIndex`]: just the `name → addr` mapping. +/// The heavy `ExprMetaArena` is parsed (to advance the cursor and +/// handle every metadata variant, e.g. `CallSite`) but discarded; +/// hints live on [`LazyIndex::hints`]. #[derive(Debug, Clone)] pub struct LazyNamed { pub name: Name, pub addr: Address, - pub hint: Option, } /// `IX_COMPILE_DEMOTE` (default **on**; set `0` to disable): store the @@ -196,6 +198,22 @@ pub struct LazyIndex { pub consts: Vec, pub named: Vec, pub blobs: Vec<(Address, Vec)>, + /// Bundle root (`Env::main`) as read from the header. + pub main: Option
, + /// Bundle trust boundary (`Env::assumptions`), in header (sorted) order. + pub assumptions: Vec
, + /// §3 anon_hints verbatim (file order) — the same content the full + /// reader puts in `Env::anon_hints`. + pub hints: Vec<(Address, ReducibilityHints)>, + /// §6 comms (tiny in practice; empty for compile-produced envs). + pub comms: Vec<(Address, Comm)>, + /// Byte offset (within the parsed buffer) of §5's entry count — where + /// a [`crate::serialize::NamedMetaCursor`] starts its streaming walk. + pub named_section_offset: usize, + /// §4 positional index → name-component address, retained so §5 + /// entries can be re-parsed standalone (`get_named_indexed`) without + /// re-walking §4. ~32 B per name. + pub name_reverse_index: crate::metadata::NameReverseIndex, } /// The Ixon environment. @@ -208,7 +226,7 @@ pub struct LazyIndex { /// - `blobs`: Raw data (strings, nats, files) /// - `names`: Hash-consed Lean.Name components (Address -> Name) /// - `comms`: Cryptographic commitments (secrets) -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Env { /// Alpha-invariant constants: Address -> LazyConstant (raw bytes + /// optional materialized cache; see [`LazyConstant`]). @@ -221,20 +239,33 @@ pub struct Env { pub names: IxonMap, /// Cryptographic commitments: commitment Address -> Comm pub comms: IxonMap, - /// Reducibility hints sidecar harvested by [`Env::get_anon`] from the - /// otherwise-discarded Named section. Keyed by the constant's - /// projection/standalone address (i.e. `Named.addr` — the address the - /// kernel sees, **not** the name-hash address). Empty for envs loaded - /// via [`Env::get`] / [`Env::new`] / `store_*`; meta-mode ingress - /// pulls hints directly from `Named.meta` and ignores this field. + /// Reducibility hints, keyed by the constant's projection/standalone + /// address (i.e. `Named.addr` — the address the kernel sees, **not** + /// the name-hash address). The SINGLE home for hints: the compiler + /// registers them here ([`Env::register_hint`]), every writer + /// serializes this map as the file's hints section, and every reader + /// populates it from that section. Hints do not appear in `Named` + /// metadata. /// - /// Anon-mode ingress passes these hints through to - /// `ingress_defn` so the kernel's lazy-delta tiebreak - /// (`def_eq::def_rank_id`) sees realistic heights instead of the - /// constant `Regular(0)` fallback. Hints are performance advice — - /// supplying them in anon mode does not relax the kernel's - /// metadata-free correctness model. - pub anon_hints: FxHashMap, + /// Ingress passes these hints through to `ingress_defn` so the + /// kernel's lazy-delta tiebreak (`def_eq::def_rank_id`) sees + /// realistic heights instead of the constant `Regular(0)` fallback. + /// Hints are performance advice — supplying them does not relax the + /// kernel's metadata-free correctness model — and are intentionally + /// NOT covered by the consts merkle root. + pub anon_hints: IxonMap, + /// Distinguished root constant for bundle envs; `None` for whole + /// environments. A pointer, not a proof: nothing in the file + /// authenticates it, so readers check `main ∈ consts` and consumers + /// holding an externally-expected address (from a Claim, a request) + /// must compare against it. + pub main: Option
, + /// Explicit trust boundary for thin bundles: addresses (constants + /// or blobs) the receiver is expected to already have, so the + /// closure of `main` need not be carried in full. Serialized as a + /// strictly ascending leaf list; `merkle_root_canonical` over it + /// reproduces the root a `Claim::assumptions` field commits to. + pub assumptions: FxHashSet
, } impl Env { @@ -245,7 +276,9 @@ impl Env { blobs: IxonMap::new(), names: IxonMap::new(), comms: IxonMap::new(), - anon_hints: FxHashMap::default(), + anon_hints: IxonMap::new(), + main: None, + assumptions: FxHashSet::default(), } } @@ -267,6 +300,38 @@ impl Env { self.blobs.get(addr).map(|r| r.clone()) } + /// Register a constant's reducibility hints (the compile-time + /// producer of [`Self::anon_hints`]; readers populate the map from + /// the file's hints section instead). + /// + /// Alpha-equivalent definitions share one constant address but may + /// carry different hints (e.g. one alias marked `@[reducible]`), and + /// parallel compile workers may race on the entry — so the winner + /// must not depend on arrival order. The merge keeps the minimum + /// under `(tag, height)` with `Opaque < Abbrev < Regular(h)`: + /// commutative, associative, idempotent, hence order-independent. + /// The Lean compiler mirror applies the same rule. + #[cfg(not(target_arch = "riscv64"))] + pub fn register_hint(&self, addr: Address, hints: ReducibilityHints) { + fn rank(h: &ReducibilityHints) -> (u8, u32) { + match h { + ReducibilityHints::Opaque => (0, 0), + ReducibilityHints::Abbrev => (1, 0), + ReducibilityHints::Regular(x) => (2, *x), + } + } + match self.anon_hints.entry(addr) { + dashmap::mapref::entry::Entry::Occupied(mut e) => { + if rank(&hints) < rank(e.get()) { + e.insert(hints); + } + }, + dashmap::mapref::entry::Entry::Vacant(e) => { + e.insert(hints); + }, + } + } + /// Store a structured constant under `addr`. /// /// Serializes the constant once. With demotion disabled, @@ -331,6 +396,86 @@ impl Env { self.consts.insert(addr, LazyConstant::from_mmap_slice(mmap, offset, len)); } + /// Build a metadata-light `Env` from a parsed [`LazyIndex`] over + /// `data` — the exact buffer handed to [`Env::parse_lazy_index`] + /// (constant windows are copied out of it, so the result owns its + /// bytes). `named` entries carry empty `ConstantMeta`; everything an + /// anon-mode consumer needs (consts, name→addr, §3 hints, blobs, + /// comms, `main`, `assumptions`) is populated. + /// + /// This is the memory-lean anon loading path (e.g. `ix diff`): the + /// full reader materializes every `ConstantMeta`, which at mathlib + /// scale costs tens of GB; this path costs roughly the consts + /// section plus the name table. Host-only — see `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn from_lazy_index( + index: &LazyIndex, + data: &[u8], + ) -> Result { + let env = Env::new(); + for c in &index.consts { + let end = Self::lazy_slice_end(c, data.len(), "from_lazy_index")?; + env.store_const_lazy(c.addr.clone(), Arc::from(&data[c.offset..end])); + } + Ok(Self::fill_from_lazy_index(env, index)) + } + + /// [`Env::from_lazy_index`] over a memory-mapped buffer: constant + /// windows stay zero-copy mmap slices (the OS page cache backs them) + /// instead of heap copies. `mmap` must be the exact buffer + /// [`Env::parse_lazy_index`] walked. + #[cfg(not(target_arch = "riscv64"))] + pub fn from_lazy_index_mmap( + index: &LazyIndex, + mmap: &Arc, + ) -> Result { + let env = Env::new(); + for c in &index.consts { + Self::lazy_slice_end(c, mmap.len(), "from_lazy_index_mmap")?; + env.store_const_lazy_mmap( + c.addr.clone(), + Arc::clone(mmap), + c.offset, + c.len, + ); + } + Ok(Self::fill_from_lazy_index(env, index)) + } + + #[cfg(not(target_arch = "riscv64"))] + fn lazy_slice_end( + c: &LazyConstSlice, + data_len: usize, + who: &str, + ) -> Result { + c.offset.checked_add(c.len).filter(|e| *e <= data_len).ok_or_else(|| { + format!( + "{who}: constant window [{}, +{}) out of bounds ({data_len} bytes)", + c.offset, c.len, + ) + }) + } + + /// The non-const parts shared by both `from_lazy_index` variants. + #[cfg(not(target_arch = "riscv64"))] + fn fill_from_lazy_index(mut env: Env, index: &LazyIndex) -> Env { + for n in &index.named { + env.register_name(n.name.clone(), Named::with_addr(n.addr.clone())); + } + for (addr, hint) in &index.hints { + env.anon_hints.insert(addr.clone(), *hint); + } + for (addr, bytes) in &index.blobs { + env.blobs.insert(addr.clone(), bytes.clone()); + } + for (addr, comm) in &index.comms { + env.comms.insert(addr.clone(), comm.clone()); + } + env.main = index.main.clone(); + env.assumptions = index.assumptions.iter().cloned().collect(); + env + } + /// Get a constant by address, materializing on demand. /// /// Returns `None` if the address is not present or materialization @@ -477,45 +622,340 @@ impl Env { v.sort_unstable(); v } -} -impl Clone for Env { - // `mut` is only needed on `riscv64` where `IxonMap` wraps `FxHashMap` and - // `insert` takes `&mut self`; on host `DashMap::insert` takes `&self`. - #[cfg_attr(not(target_arch = "riscv64"), allow(unused_mut))] - fn clone(&self) -> Self { - let mut consts = IxonMap::new(); - for entry in self.consts.iter() { - consts.insert(entry.key().clone(), entry.value().clone()); + /// Collect `c`'s outgoing closure edges: Expr-level `refs`, a + /// projection's structural `block` pointer, and — for a `Muts` + /// block at `addr` — every member/constructor projection address + /// (derived, not stored anywhere; `ingress_anon_block` computes + /// these and requires them present in the env). + fn closure_edges(addr: &Address, c: &Constant, edges: &mut Vec
) { + edges.extend(c.refs.iter().cloned()); + match &c.info { + ConstantInfo::IPrj(p) => edges.push(p.block.clone()), + ConstantInfo::CPrj(p) => edges.push(p.block.clone()), + ConstantInfo::RPrj(p) => edges.push(p.block.clone()), + ConstantInfo::DPrj(p) => edges.push(p.block.clone()), + ConstantInfo::Muts(members) => { + for (i, m) in members.iter().enumerate() { + let i = i as u64; + edges.push(match m { + MutConst::Defn(_) => defn_proj_address(i, addr), + MutConst::Indc(_) => indc_proj_address(i, addr), + MutConst::Recr(_) => recr_proj_address(i, addr), + }); + if let MutConst::Indc(ind) = m { + for cidx in 0..ind.ctors.len() as u64 { + edges.push(ctor_proj_address(i, cidx, addr)); + } + } + } + }, + _ => {}, } + } - let mut named = IxonMap::new(); - for entry in self.named.iter() { - named.insert(entry.key().clone(), entry.value().clone()); + /// BFS-collect the full dependency closure of `roots`, following + /// all three structural edge kinds (see [`Self::closure_edges`]): + /// + /// 1. `Constant.refs` — Expr-level references (constants and blobs); + /// 2. projection → its `Muts` block (`DPrj`/`IPrj`/`RPrj`/`CPrj` + /// carry `block` in the info payload and have EMPTY refs tables, + /// so a refs-only walk returns just the projection itself); + /// 3. `Muts` block → every member/constructor projection address. + /// + /// Compare [`Self::bfs_refs`], which follows only edge 1 and stays + /// the basis of claim assumption roots. The kernel's + /// `anon_work::closure_addrs` delegates here. + /// + /// Referenced addresses absent from `consts` (blobs and external + /// assumptions) appear in the returned set as leaves. + pub fn bfs_closure(&self, roots: &[Address]) -> FxHashSet
{ + let mut closure: FxHashSet
= FxHashSet::default(); + let mut queue: VecDeque
= VecDeque::new(); + for r in roots { + if closure.insert(r.clone()) { + queue.push_back(r.clone()); + } + } + while let Some(addr) = queue.pop_front() { + // Materialize just long enough to read edges; drop the map + // guard before recursing (see `bfs_refs`). + let constant = self.consts.get(&addr).and_then(|r| r.value().get().ok()); + let Some(c) = constant else { continue }; + let mut edges: Vec
= Vec::new(); + Self::closure_edges(&addr, &c, &mut edges); + for e in edges { + if closure.insert(e.clone()) { + queue.push_back(e); + } + } } + closure + } - let mut blobs = IxonMap::new(); - for entry in self.blobs.iter() { - blobs.insert(entry.key().clone(), entry.value().clone()); + /// Bundle completeness check (receiver side): `main` must be a + /// stored constant, and every address reachable from it via + /// [`Self::bfs_closure`] must be either carried by this env + /// (consts ∪ blobs) or declared in `assumptions`. + /// + /// This validates the VALUE pin. Display-metadata completeness + /// (names, `DataValue` blobs) is the writer's business — + /// [`Self::prune_to_closure`] carries it; a receiver that only + /// typechecks/evaluates never needs it. + pub fn validate_closed(&self) -> Result<(), String> { + let Some(main) = self.main.clone() else { + return Err("validate_closed: env has no main".to_string()); + }; + if self.consts.get(&main).is_none() { + return Err(format!( + "validate_closed: main {} not present in consts", + main.hex() + )); } + for addr in self.bfs_closure(std::slice::from_ref(&main)) { + if self.consts.contains_key(&addr) + || self.blobs.contains_key(&addr) + || self.assumptions.contains(&addr) + { + continue; + } + return Err(format!( + "validate_closed: {} reachable from main but neither carried nor \ + assumed", + addr.hex() + )); + } + Ok(()) + } - let mut names = IxonMap::new(); - for entry in self.names.iter() { - names.insert(entry.key().clone(), entry.value().clone()); + /// Build a self-contained bundle env: the 3-edge closure of `main`, + /// cut at `assumed`. + /// + /// - Reached constants are carried with their GENUINE bytes + /// (`store_const_lazy`), so the receiver's per-entry hash check + /// and consts merkle root hold; reached blobs are copied. + /// - Cut-points actually reached go to `out.assumptions` (minimal: + /// a declared-but-unreached assumption is not carried). + /// - `anon_hints` are copied per carried constant (else a bundle + /// regresses kernel-check time to the `Regular(0)` fallback). + /// - Display metadata is carried for every carried constant: its + /// `named` entries (all of them — alpha-equivalent names may + /// share one address), the name components they reference (with + /// full parent chains and string-component blobs), `DataValue` + /// payload blobs, `meta_refs` extension DAG edges, and aux_gen + /// `original` constants. Metadata can introduce new DAG edges, so + /// the walk runs to fixpoint (usually two rounds). + /// + /// Errors if a reached address is in neither `consts`, `blobs`, nor + /// `assumed` — the source env cannot produce a closed bundle for + /// `main` under that cut. + /// + /// Host-only — see `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure( + &self, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; + let mut named_done: FxHashSet = FxHashSet::default(); + loop { + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; + + // ── Named pass: carry display metadata for every carried + // constant. Metadata references content the value walk cannot + // see; new DAG edges feed the next value pass. + for entry in self.named.iter() { + let (name, named) = (entry.key(), entry.value()); + if !out.consts.contains_key(&named.addr) || named_done.contains(name) { + continue; + } + named_done.insert(name.clone()); + Self::carry_named_entry( + &mut out, + name, + named, + &|na| self.get_name(na), + &|ba| self.get_blob(ba), + &mut visited, + &mut pending, + )?; + } + + // The named pass ran against the final consts of this round; if + // it produced no new DAG work, the walk is complete. + if pending.is_empty() { + break; + } } + Ok(out) + } + + /// Value-only bundle: the 3-edge closure of `main` cut at `assumed` — + /// constants (genuine bytes), value blobs, per-constant hints, + /// `main`, and reached assumptions. No display metadata at all: + /// `names`/`named` stay empty (§4/§5 serialize as empty sections). + /// The result still passes [`Self::validate_closed`] (which checks + /// the value pin only) — this is the minimal artifact a receiver + /// needs to typecheck/evaluate the pinned value. Host-only — see + /// `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure_anon( + &self, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; + Ok(out) + } - let mut comms = IxonMap::new(); - for entry in self.comms.iter() { - comms.insert(entry.key().clone(), entry.value().clone()); + /// Shared prune setup: assumed-main guard, `out` with `main` set, + /// seeded worklist. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn prune_init( + main: &Address, + assumed: &FxHashSet
, + ) -> Result<(Env, FxHashSet
, VecDeque
), String> { + if assumed.contains(main) { + return Err("prune_to_closure: main cannot be assumed".to_string()); } + let mut out = Env::new(); + out.main = Some(main.clone()); + let mut visited: FxHashSet
= FxHashSet::default(); + let mut pending: VecDeque
= VecDeque::new(); + visited.insert(main.clone()); + pending.push_back(main.clone()); + Ok((out, visited, pending)) + } - Env { - consts, - named, - blobs, - names, - comms, - anon_hints: self.anon_hints.clone(), + /// One value pass: 3-edge BFS over pending roots, cut at `assumed`. + /// Carries constant bytes + per-constant hints + reached blobs; + /// records reached cut points in `out.assumptions`. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn prune_value_pass( + &self, + out: &mut Env, + visited: &mut FxHashSet
, + pending: &mut VecDeque
, + assumed: &FxHashSet
, + ) -> Result<(), String> { + while let Some(addr) = pending.pop_front() { + if assumed.contains(&addr) { + out.assumptions.insert(addr); + continue; + } + if let Some(bytes) = self.get_const_bytes(&addr) { + out.store_const_lazy(addr.clone(), bytes); + if let Some(h) = self.anon_hints.get(&addr) { + out.anon_hints.insert(addr.clone(), *h); + } + let c = self.get_const(&addr).ok_or_else(|| { + format!("prune_to_closure: constant {} unparseable", addr.hex()) + })?; + let mut edges: Vec
= Vec::new(); + Self::closure_edges(&addr, &c, &mut edges); + for e in edges { + if visited.insert(e.clone()) { + pending.push_back(e); + } + } + } else if let Some(blob) = self.get_blob(&addr) { + out.blobs.insert(addr, blob); + } else { + return Err(format!( + "prune_to_closure: {} reachable from main but not in \ + consts/blobs and not assumed", + addr.hex() + )); + } + } + Ok(()) + } + + /// Carry one named entry and its metadata dependencies into `out`: + /// the `Named` row, its name's component chain, every + /// metadata-referenced name component and blob, and (via + /// `visited`/`pending`) any new constant DAG edges (aux `original`s, + /// `meta_refs`) for the next value pass. `resolve_name`/`get_blob` + /// abstract the source: the in-memory env for + /// [`Self::prune_to_closure`]; the §4 lookup + lazy env for the + /// streaming variant (`Env::prune_to_closure_streaming` in + /// `serialize.rs`) — one body, so the two paths cannot drift. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn carry_named_entry( + out: &mut Env, + name: &Name, + named: &Named, + resolve_name: &dyn Fn(&Address) -> Option, + get_blob: &dyn Fn(&Address) -> Option>, + visited: &mut FxHashSet
, + pending: &mut VecDeque
, + ) -> Result<(), String> { + out.named.insert(name.clone(), named.clone()); + Self::carry_name(out, name); + + let mut name_addrs: Vec
= Vec::new(); + let mut blob_addrs: Vec
= Vec::new(); + let mut dag_addrs: Vec
= Vec::new(); + named.meta().collect_deps(&mut name_addrs, &mut blob_addrs, &mut dag_addrs); + if let Some((orig_addr, orig_meta)) = named.original() { + dag_addrs.push(orig_addr); + orig_meta.collect_deps(&mut name_addrs, &mut blob_addrs, &mut dag_addrs); + } + for na in name_addrs { + let Some(name) = resolve_name(&na) else { + return Err(format!( + "prune_to_closure: metadata references name {} absent from names", + na.hex() + )); + }; + Self::carry_name(out, &name); + } + for ba in blob_addrs { + let Some(blob) = get_blob(&ba) else { + return Err(format!( + "prune_to_closure: metadata references blob {} absent from blobs", + ba.hex() + )); + }; + out.blobs.insert(ba, blob); + } + for da in dag_addrs { + if visited.insert(da.clone()) { + pending.push_back(da); + } + } + Ok(()) + } + + /// Copy `name` and its full parent chain into `out.names`, storing + /// each string component's bytes as a blob (the compiler's + /// convention — mirrors `addNameComponentsWithBlobs` on the Lean + /// side). Stops early once a component is already present: its + /// parents were carried with it. + #[cfg(not(target_arch = "riscv64"))] + fn carry_name(out: &mut Env, name: &Name) { + use ix_common::env::NameData; + let mut cur = name.clone(); + loop { + let addr = Address::from_blake3_hash(*cur.get_hash()); + if out.names.get(&addr).is_some() { + return; + } + out.names.insert(addr, cur.clone()); + let next = match cur.as_data() { + NameData::Anonymous(_) => None, + NameData::Str(parent, s, _) => { + out.store_blob(s.as_bytes().to_vec()); + Some(parent.clone()) + }, + NameData::Num(parent, _, _) => Some(parent.clone()), + }; + match next { + Some(p) => cur = p, + None => return, + } } } } @@ -912,4 +1352,313 @@ mod tests { ); } } + + // --------------------------------------------------------------------------- + // bfs_closure / prune_to_closure / validate_closed + // --------------------------------------------------------------------------- + + fn defn_member(discriminator: u64) -> crate::constant::Definition { + use crate::constant::{DefKind, Definition}; + use ix_common::env::DefinitionSafety; + Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls: discriminator, + typ: Arc::new(Expr::Sort(0)), + value: Arc::new(Expr::Var(0)), + } + } + + #[test] + fn bfs_closure_follows_projection_and_block_edges() { + use crate::constant::{MutConst, defn_proj_address, defn_proj_constant}; + let env = Env::new(); + let block_addr = store_canonical( + &env, + Constant::new(ConstantInfo::Muts(vec![ + MutConst::Defn(defn_member(0)), + MutConst::Defn(defn_member(1)), + ])), + ); + // Member projections live at derived addresses. + let p0_addr = + store_canonical(&env, defn_proj_constant(0, block_addr.clone())); + assert_eq!(p0_addr, defn_proj_address(0, &block_addr)); + let p1_addr = + store_canonical(&env, defn_proj_constant(1, block_addr.clone())); + + // From a projection root: the block and its sibling projections + // are reachable via the structural edges. + let closure = env.bfs_closure(std::slice::from_ref(&p0_addr)); + assert!(closure.contains(&p0_addr)); + assert!(closure.contains(&block_addr), "Prj → block edge"); + assert!(closure.contains(&p1_addr), "block → sibling projection edge"); + + // A refs-only walk sees none of this (projections have empty refs). + let refs_only = env.bfs_refs(&p0_addr); + assert_eq!(refs_only.len(), 1, "refs-only walk stops at the projection"); + } + + #[test] + fn prune_to_closure_carries_value_closure_and_blobs() { + let env = Env::new(); + let blob_addr = env.store_blob(b"forty two".to_vec()); + let c = store_canonical(&env, const_with_refs(vec![])); + let a = store_canonical( + &env, + const_with_refs(vec![blob_addr.clone(), c.clone()]), + ); + let d = store_canonical(&env, const_with_refs_discriminator(vec![], 7)); + + let bundle = env.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + assert_eq!(bundle.main, Some(a.clone())); + assert!(bundle.consts.contains_key(&a)); + assert!(bundle.consts.contains_key(&c)); + assert!(!bundle.consts.contains_key(&d), "unreachable const not carried"); + assert_eq!(bundle.get_blob(&blob_addr), Some(b"forty two".to_vec())); + assert!(bundle.assumptions.is_empty()); + bundle.validate_closed().unwrap(); + + // Bundle survives a serialize → deserialize roundtrip closed + // (genuine bytes: per-entry hashes and the merkle root hold). + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let loaded = Env::get(&mut buf.as_slice()).unwrap(); + loaded.validate_closed().unwrap(); + assert_eq!(loaded.main, Some(a)); + } + + #[test] + fn prune_to_closure_cuts_at_assumed() { + let env = Env::new(); + let c = store_canonical(&env, const_with_refs(vec![])); + let b = store_canonical(&env, const_with_refs(vec![c.clone()])); + let a = store_canonical(&env, const_with_refs(vec![b.clone()])); + let assumed: FxHashSet
= [b.clone()].into_iter().collect(); + let bundle = env.prune_to_closure(&a, &assumed).unwrap(); + assert!(bundle.consts.contains_key(&a)); + assert!(!bundle.consts.contains_key(&b), "cut point not carried"); + assert!(!bundle.consts.contains_key(&c), "beyond the cut not carried"); + assert_eq!( + bundle.assumptions, assumed, + "minimal assumptions = reached cuts" + ); + bundle.validate_closed().unwrap(); + } + + #[test] + fn prune_to_closure_missing_dep_errors_unless_assumed() { + let env = Env::new(); + let ghost = Address::hash(b"ghost"); + let a = store_canonical(&env, const_with_refs(vec![ghost.clone()])); + let err = env.prune_to_closure(&a, &FxHashSet::default()).unwrap_err(); + assert!(err.contains("not in consts/blobs"), "got: {err}"); + let assumed: FxHashSet
= [ghost].into_iter().collect(); + let bundle = env.prune_to_closure(&a, &assumed).unwrap(); + bundle.validate_closed().unwrap(); + } + + #[test] + fn validate_closed_rejects_missing_blob_unless_assumed() { + let mut env = Env::new(); + let blob_addr = Address::hash(b"blob-bytes"); + let a = store_canonical(&env, const_with_refs(vec![blob_addr.clone()])); + env.main = Some(a); + let err = env.validate_closed().unwrap_err(); + assert!(err.contains("reachable from main"), "got: {err}"); + env.assumptions.insert(blob_addr); + env.validate_closed().unwrap(); + } + + #[test] + fn prune_to_closure_carries_named_metadata() { + use crate::metadata::{ConstantMetaInfo, ExprMeta}; + let env = Env::new(); + let a = store_canonical(&env, const_with_refs(vec![])); + // A blob referenced ONLY from metadata (meta_refs extension table) + // — invisible to the value walk. + let meta_blob = env.store_blob(b"callsite payload".to_vec()); + let name = n("Bundled"); + let name_addr = Address::from_blake3_hash(*name.get_hash()); + env.store_name(name_addr.clone(), name.clone()); + let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { + name: name_addr.clone(), + lvls: vec![], + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }); + meta.meta_refs.push(meta_blob.clone()); + env.register_name(name.clone(), Named::new(a.clone(), meta)); + env.register_hint(a.clone(), ReducibilityHints::Regular(3)); + + let bundle = env.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + assert!(bundle.named.get(&name).is_some(), "named entry carried"); + assert_eq!( + bundle.get_name(&name_addr), + Some(name.clone()), + "name component carried" + ); + assert_eq!( + bundle.get_blob(&meta_blob), + Some(b"callsite payload".to_vec()), + "meta_refs blob carried" + ); + // String-component blob carried too (compiler convention). + assert_eq!( + bundle.get_blob(&Address::hash(b"Bundled")), + Some(b"Bundled".to_vec()) + ); + // Hints are carried per constant by the prune; the anon reader + // sees them from the hints section without touching metadata. + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!( + anon.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(3)) + ); + } + + /// Fixture for the streaming/anon prune tests: `a` (the main) named + /// "Foo" with Def metadata carrying (i) a §4-resolved non-ancestor + /// name component (`u`, in `lvls`), (ii) a `meta_refs` blob, and + /// (iii) an `original` edge to `b` — a constant NOT reachable from + /// `a`'s value, so carrying it requires a second fixpoint round; `b` + /// is named "Bar". Returns (serialized env, a, b, "Bar"). + fn streaming_prune_fixture() -> (Vec, Address, Address, Name) { + use crate::metadata::{ConstantMetaInfo, ExprMeta}; + let env = Env::new(); + let a = store_canonical(&env, const_with_refs(vec![])); + let b = store_canonical(&env, const_with_refs(vec![a.clone()])); + + let foo = n("Foo"); + let foo_addr = Address::from_blake3_hash(*foo.get_hash()); + env.store_name(foo_addr.clone(), foo.clone()); + let u = n("u"); + let u_addr = Address::from_blake3_hash(*u.get_hash()); + env.store_name(u_addr.clone(), u.clone()); + let meta_blob = env.store_blob(b"payload".to_vec()); + let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { + name: foo_addr, + lvls: vec![u_addr], + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }); + meta.meta_refs.push(meta_blob); + let mut foo_named = Named::new(a.clone(), meta); + foo_named.set_original(b.clone(), ConstantMeta::default()); + env.register_name(foo, foo_named); + env.register_hint(a.clone(), ReducibilityHints::Regular(2)); + let bar = n("Bar"); + let bar_addr = Address::from_blake3_hash(*bar.get_hash()); + env.store_name(bar_addr, bar.clone()); + env.register_name(bar.clone(), Named::with_addr(b.clone())); + + let mut bytes = Vec::new(); + env.put(&mut bytes).unwrap(); + (bytes, a, b, bar) + } + + /// The streaming prune (lazy env + §5 re-stream per fixpoint round) + /// must produce a byte-identical bundle to the in-memory prune over + /// the full reader — including metadata that forces a second round. + #[test] + fn prune_streaming_matches_full_byte_identical() { + let (bytes, a, b, bar) = streaming_prune_fixture(); + let full = { + let mut cur = bytes.as_slice(); + Env::get(&mut cur).unwrap() + }; + let (index, names) = Env::parse_lazy_index_with_names(&bytes).unwrap(); + let lazy = Env::from_lazy_index(&index, &bytes).unwrap(); + let ser = |e: &Env| { + let mut v = Vec::new(); + e.put(&mut v).unwrap(); + v + }; + + let via_full = full.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + let via_stream = lazy + .prune_to_closure_streaming( + &index, + &bytes, + &names, + &a, + &FxHashSet::default(), + ) + .unwrap(); + // Round-2 evidence: Bar rides in through Foo's `original` edge. + assert!(via_stream.named.get(&bar).is_some(), "round-2 named carried"); + assert!(via_stream.consts.contains_key(&b)); + via_stream.validate_closed().unwrap(); + assert_eq!( + ser(&via_full), + ser(&via_stream), + "streaming bundle must be byte-identical to the full-reader bundle" + ); + + // With `b` assumed: the original edge lands in assumptions, Bar is + // not carried — parity still holds. + let assumed: FxHashSet
= [b.clone()].into_iter().collect(); + let via_full_cut = full.prune_to_closure(&a, &assumed).unwrap(); + let via_stream_cut = lazy + .prune_to_closure_streaming(&index, &bytes, &names, &a, &assumed) + .unwrap(); + assert!(via_stream_cut.named.get(&bar).is_none()); + assert!(via_stream_cut.assumptions.contains(&b)); + assert_eq!(ser(&via_full_cut), ser(&via_stream_cut)); + } + + /// `prune_to_closure_anon` carries the value closure only: no names, + /// no named entries, no metadata-edge constants or blobs — the + /// minimal typecheck/eval artifact. Hints (§3) still ride; the + /// bundle validates closed and round-trips with empty §4/§5. + #[test] + fn prune_anon_value_only() { + let (bytes, a, b, _) = streaming_prune_fixture(); + let (index, _) = Env::parse_lazy_index_with_names(&bytes).unwrap(); + let lazy = Env::from_lazy_index(&index, &bytes).unwrap(); + + let bundle = lazy.prune_to_closure_anon(&a, &FxHashSet::default()).unwrap(); + assert!( + bundle.named.is_empty(), + "no named entries: {}", + bundle.named.len() + ); + assert!(bundle.names.is_empty(), "no name components"); + assert!(bundle.blobs.is_empty(), "no metadata/name-string blobs"); + assert!(bundle.consts.contains_key(&a)); + assert!( + !bundle.consts.contains_key(&b), + "metadata-only edges must not be walked in anon mode" + ); + // Hints ride the lazy env's anon_hints into the bundle. + assert_eq!( + bundle.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(2)) + ); + bundle.validate_closed().unwrap(); + + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let back = { + let mut cur = buf.as_slice(); + Env::get(&mut cur).unwrap() + }; + assert!(back.named.is_empty(), "no §5 entries after roundtrip"); + // The writer always emits the anonymous name as §4 entry 0. + assert!(back.names.len() <= 1, "§4 carries at most the anon entry"); + assert_eq!(back.main, Some(a.clone())); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!( + anon.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(2)) + ); + } } diff --git a/crates/ixon/src/lib.rs b/crates/ixon/src/lib.rs index 84a13addb..bb9a900e1 100644 --- a/crates/ixon/src/lib.rs +++ b/crates/ixon/src/lib.rs @@ -9,6 +9,8 @@ pub mod assumption_tree; pub mod comm; pub mod constant; +#[cfg(not(target_arch = "riscv64"))] +pub mod diff; pub mod env; pub mod error; pub mod expr; @@ -29,6 +31,11 @@ pub use constant::{ Definition, DefinitionProj, Inductive, InductiveProj, MutConst, Quotient, Recursor, RecursorProj, RecursorRule, }; +#[cfg(not(target_arch = "riscv64"))] +pub use diff::{ + DiffPhase, EnvDiff, EnvStats, JoinProgress, LazySide, NamedChange, + diff_env_bytes, diff_envs, diff_envs_lazy, diff_envs_with, +}; pub use env::{Env, Named}; pub use error::{CompileError, DecompileError, SerializeError}; pub use expr::Expr; diff --git a/crates/ixon/src/map.rs b/crates/ixon/src/map.rs index 748ebafb1..5d514782f 100644 --- a/crates/ixon/src/map.rs +++ b/crates/ixon/src/map.rs @@ -20,7 +20,7 @@ mod riscv_impl { use rustc_hash::FxHashMap; - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct IxonMap(FxHashMap); impl Default for IxonMap { diff --git a/crates/ixon/src/metadata.rs b/crates/ixon/src/metadata.rs index c126fafb0..315be9750 100644 --- a/crates/ixon/src/metadata.rs +++ b/crates/ixon/src/metadata.rs @@ -121,7 +121,6 @@ pub enum ConstantMetaInfo { Def { name: Address, lvls: Vec
, - hints: ReducibilityHints, all: Vec
, ctx: Vec
, arena: ExprMeta, @@ -252,6 +251,105 @@ impl ConstantMeta { || !self.meta_univs.is_empty() } + /// Enumerate every external address this metadata references, + /// partitioned by the table that resolves it: + /// + /// - `names`: name-component addresses (resolved against + /// `Env.names`) — the variant's `name`/`lvls`/`all`/`ctx`/... + /// fields, arena binder/ref/proj/call-site names, and KVMap keys + /// plus `DataValue::OfName` payloads; + /// - `blobs`: raw-byte payload addresses (`DataValue` + /// strings/nats/ints/syntax, resolved against `Env.blobs`); + /// - `dag`: `meta_refs` extension-table addresses — constants or + /// blobs referenced by collapsed call-site argument expressions, + /// i.e. genuine value-DAG edges the primary `Constant.refs` walk + /// cannot see. + /// + /// Used by `Env::prune_to_closure` to carry a bundle's display + /// metadata completely. Duplicates are not filtered; callers dedup. + pub fn collect_deps( + &self, + names: &mut Vec
, + blobs: &mut Vec
, + dag: &mut Vec
, + ) { + use ConstantMetaInfo as I; + let mut arena: Option<&ExprMeta> = None; + match &self.info { + I::Empty => {}, + I::Def { name, lvls, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Axio { name, lvls, arena: a, .. } + | I::Quot { name, lvls, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + arena = Some(a); + }, + I::Indc { name, lvls, ctors, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(ctors.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Ctor { name, lvls, induct, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.push(induct.clone()); + arena = Some(a); + }, + I::Rec { name, lvls, rules, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(rules.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Muts { all, .. } => { + for class in all { + names.extend(class.iter().cloned()); + } + }, + } + if let Some(a) = arena { + for node in &a.nodes { + match node { + ExprMetaData::Leaf | ExprMetaData::App { .. } => {}, + ExprMetaData::Binder { name, .. } + | ExprMetaData::LetBinder { name, .. } + | ExprMetaData::Ref { name } + | ExprMetaData::CallSite { name, .. } => names.push(name.clone()), + ExprMetaData::Prj { struct_name, .. } => { + names.push(struct_name.clone()); + }, + ExprMetaData::Mdata { mdata, .. } => { + for kv in mdata { + for (key, value) in kv { + names.push(key.clone()); + match value { + DataValue::OfName(a) => names.push(a.clone()), + DataValue::OfString(a) + | DataValue::OfNat(a) + | DataValue::OfInt(a) + | DataValue::OfSyntax(a) => blobs.push(a.clone()), + DataValue::OfBool(_) => {}, + } + } + } + }, + } + } + } + dag.extend(self.meta_refs.iter().cloned()); + } + /// Delegate indexed serialization to the inner enum, then serialize /// extension tables. pub fn put_with( @@ -1055,20 +1153,10 @@ impl ConstantMetaInfo { ) -> Result<(), String> { match self { Self::Empty => put_u8(255, buf), - Self::Def { - name, - lvls, - hints, - all, - ctx, - arena, - type_root, - value_root, - } => { + Self::Def { name, lvls, all, ctx, arena, type_root, value_root } => { put_u8(0, buf); put_idx(name, idx, buf)?; put_idx_vec(lvls, idx, buf)?; - hints.put_ser(buf); put_idx_vec(all, idx, buf)?; put_idx_vec(ctx, idx, buf)?; arena.put_with(idx, buf)?; @@ -1161,7 +1249,6 @@ impl ConstantMetaInfo { 0 => Ok(Self::Def { name: get_idx(buf, rev)?, lvls: get_idx_vec(buf, rev)?, - hints: ReducibilityHints::get_ser(buf)?, all: get_idx_vec(buf, rev)?, ctx: get_idx_vec(buf, rev)?, arena: ExprMeta::get_with(buf, rev)?, @@ -1300,7 +1387,6 @@ mod tests { let meta = ConstantMeta::new(ConstantMetaInfo::Def { name: addr1.clone(), lvls: vec![addr2.clone(), addr3.clone()], - hints: ReducibilityHints::Regular(10), all: vec![addr1.clone()], ctx: vec![addr2.clone()], arena, diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index 08ee0856e..e4981eeaf 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -76,25 +76,143 @@ fn put_address(a: &Address, buf: &mut Vec) { put_bytes(a.as_bytes(), buf); } -/// Read the optional trailing `anon_hints` section written by `Env::put`. -/// No-op when no bytes remain after the comms section — backward-compatible -/// with `.ixe` that have no hints or predate the section. Inserts into -/// `env.anon_hints` (overwriting any value harvested from a Named section, -/// with the identical value, so the order of harvest vs. section is moot). -fn read_anon_hints_section( +/// Write an `Option
` as `[0x00]` (None) or `[0x01][addr:32]` +/// (Some). Mirrors the Claim-side encoding in `proof.rs`. +fn put_opt_addr(opt: &Option
, buf: &mut Vec) { + match opt { + None => buf.push(0x00), + Some(addr) => { + buf.push(0x01); + buf.extend_from_slice(addr.as_bytes()); + }, + } +} + +fn get_opt_addr(buf: &mut &[u8]) -> Result, String> { + match get_u8(buf)? { + 0x00 => Ok(None), + 0x01 => Ok(Some(get_address(buf)?)), + b => Err(format!("get_opt_addr: invalid tag 0x{:02X}", b)), + } +} + +/// Read the `.ixe` header shared by every Env reader: `Tag4(0xE, 0)`, +/// the 32-byte consts merkle root, the bundle `main` pointer, and the +/// strictly ascending assumptions list. Centralized so the four readers +/// (`get`, `get_anon`, `get_anon_mmap`, `parse_lazy_index`) cannot +/// drift. `ctx` labels errors with the calling reader. +fn read_env_header( buf: &mut &[u8], - env: &mut Env, -) -> Result<(), String> { - if buf.is_empty() { - return Ok(()); + ctx: &str, +) -> Result<(Address, Option
, Vec
), String> { + let tag = Tag4::get(buf)?; + if tag.flag != Env::FLAG { + return Err(format!( + "{ctx}: expected flag 0x{:X}, got 0x{:X}", + Env::FLAG, + tag.flag + )); + } + if tag.size != 0 { + return Err(format!("{ctx}: expected Env variant 0, got {}", tag.size)); + } + let stored_root = get_address(buf)?; + // A pre-bundle-format `.ixe` has the §1 blob count here, so this + // byte is that count's Tag0 head — flag the likely cause when it + // isn't a valid opt tag. (.ixe files are regenerated artifacts.) + let main = get_opt_addr(buf).map_err(|e| { + format!( + "{ctx}: {e} in bundle header — possibly a pre-bundle-format .ixe; \ + recompile it" + ) + })?; + let n = get_u64(buf)? as usize; + // Each assumption is 32 bytes; a count the remaining buffer cannot + // hold is corruption (or a stale format) — reject before allocating. + if n > buf.len() / 32 { + return Err(format!( + "{ctx}: assumption count {n} exceeds remaining buffer — corrupt or \ + pre-bundle-format .ixe" + )); + } + let mut assumptions: Vec
= Vec::with_capacity(n); + for i in 0..n { + let addr = get_address(buf)?; + if let Some(prev) = assumptions.last() + && *prev >= addr + { + return Err(format!( + "{ctx}: assumptions not strictly ascending at idx {i} ({} then {})", + prev.hex(), + addr.hex() + )); + } + assumptions.push(addr); + } + Ok((stored_root, main, assumptions)) +} + +/// Read the §1 blob section, verifying `Address::hash(bytes) == addr` +/// per entry. Without the check a swapped blob would silently change a +/// Nat/String literal's value under an otherwise-valid file (the consts +/// merkle root covers only const addresses). +fn read_blob_section( + buf: &mut &[u8], + ctx: &str, +) -> Result)>, String> { + let num_blobs = get_u64(buf)? as usize; + // Each blob entry needs at least addr (32) + a length byte. + if num_blobs > buf.len() / 33 { + return Err(format!( + "{ctx}: blob count {num_blobs} exceeds remaining buffer" + )); + } + let mut blobs = Vec::with_capacity(num_blobs); + for i in 0..num_blobs { + let addr = get_address(buf)?; + let len = get_u64(buf)? as usize; + if buf.len() < len { + return Err(format!( + "{ctx}: need {} bytes for blob, have {}", + len, + buf.len() + )); + } + let (bytes, rest) = buf.split_at(len); + *buf = rest; + let computed = Address::hash(bytes); + if computed != addr { + return Err(format!( + "{ctx}: blob at idx {i} bytes hash to {} but stored under {}", + computed.hex(), + addr.hex() + )); + } + blobs.push((addr, bytes.to_vec())); } - let n = get_u64(buf)?; + Ok(blobs) +} + +/// Read the §3 `anon_hints` section (unconditional — every writer +/// emits it, deriving the entries from Named metadata when the +/// in-memory map is empty; see `Env::put`). +fn read_hints_section( + buf: &mut &[u8], +) -> Result, String> { + let n = get_u64(buf)? as usize; + // Each hint entry needs at least addr (32) + one byte of hint. + if n > buf.len() / 33 { + return Err(format!( + "read_hints_section: hint count {n} exceeds remaining buffer" + )); + } + let mut hints = Vec::with_capacity(n); for _ in 0..n { let addr = get_address(buf)?; - let hints = ReducibilityHints::get_ser(buf)?; - env.anon_hints.insert(addr, hints); + let hint = ReducibilityHints::get_ser(buf)?; + hints.push((addr, hint)); } - Ok(()) + Ok(hints) } fn get_address(buf: &mut &[u8]) -> Result { @@ -881,6 +999,9 @@ impl Constant { use bignat::Nat; use ix_common::env::{Name, NameData}; use rustc_hash::FxHashMap; +// Used only by the host-gated streaming prune's signature. +#[cfg(not(target_arch = "riscv64"))] +use rustc_hash::FxHashSet; /// Serialize a Name to bytes (full recursive serialization, for standalone use). pub fn put_name(name: &Name, buf: &mut Vec) { @@ -1097,6 +1218,62 @@ pub fn get_named_indexed( Ok(named) } +/// Streaming cursor over an env's §5 named entries: parse one `Named` +/// at a time against this file's own §4 reverse index, hand it out, +/// drop it. Entries arrive in the file's canonical ascending +/// name-hash order (which is exactly `Name`'s `Ord`), so two cursors +/// merge-join by name. +/// +/// This exists for the diff meta sweep: raw §5 windows are NOT +/// comparable across files (metadata name references are file-relative +/// §4 indices — identical metadata serializes to different bytes over +/// different name tables), but parsed [`Named`] values are +/// Address-valued and file-independent. Streaming keeps resident +/// metadata O(1) instead of the full reader's everything-at-once. +pub struct NamedMetaCursor<'a> { + buf: &'a [u8], + remaining: u64, + rev: &'a NameReverseIndex, +} + +impl<'a> NamedMetaCursor<'a> { + /// Open a cursor at `index.named_section_offset` within `data` — the + /// exact buffer [`Env::parse_lazy_index`] produced `index` from. + pub fn open( + data: &'a [u8], + index: &'a LazyIndex, + ) -> Result, String> { + let mut buf = data.get(index.named_section_offset..).ok_or_else(|| { + format!( + "NamedMetaCursor: §5 offset {} out of bounds ({} bytes)", + index.named_section_offset, + data.len() + ) + })?; + let remaining = get_u64(&mut buf)?; + if remaining as usize != index.named.len() { + return Err(format!( + "NamedMetaCursor: §5 count {} disagrees with the index ({} entries)", + remaining, + index.named.len() + )); + } + Ok(NamedMetaCursor { buf, remaining, rev: &index.name_reverse_index }) + } + + /// Parse the next entry: `(name-component hash, Named)`; `None` once + /// the section is exhausted. + pub fn next_entry(&mut self) -> Result, String> { + if self.remaining == 0 { + return Ok(None); + } + self.remaining -= 1; + let name_addr = get_address(&mut self.buf)?; + let named = get_named_indexed(&mut self.buf, self.rev)?; + Ok(Some((name_addr, named))) + } +} + // ============================================================================ // Env serialization // ============================================================================ @@ -1153,12 +1330,32 @@ impl Env { let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, buf); + // ───────────────────────────────────────────────────────────────────── + // Bundle header fields: distinguished root + assumed-present list + // (see `Env::main` / `Env::assumptions`). Writer-side sanity: a + // `main` outside `consts` would produce a file every reader + // rejects — fail here with the clearer message. + // ───────────────────────────────────────────────────────────────────── + if let Some(m) = &self.main + && self.consts.get(m).is_none() + { + return Err(format!("Env::put: main {} not present in consts", m.hex())); + } + put_opt_addr(&self.main, buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, buf); + for addr in &assumption_addrs { + put_address(addr, buf); + } + // ───────────────────────────────────────────────────────────────────── // Section 1: Blobs (Address -> bytes) // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 1/5 blobs: {} entries", self.blobs.len(),); + eprintln!("[Env::put] section 1/6 blobs: {} entries", self.blobs.len(),); } let mut blob_addrs: Vec
= self.blobs.iter().map(|e| e.key().clone()).collect(); @@ -1177,7 +1374,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 1/5 blobs done in {:.1}s ({} bytes so far)", + "[Env::put] section 1/6 blobs done in {:.1}s ({} bytes so far)", sec_start.elapsed().as_secs_f64(), buf.len(), ); @@ -1191,11 +1388,11 @@ impl Env { // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 2/5 consts: {} entries", self.consts.len(),); + eprintln!("[Env::put] section 2/6 consts: {} entries", self.consts.len(),); } if !quiet { eprintln!( - "[Env::put] section 2/5 consts: collected+sorted in {:.1}s, \ + "[Env::put] section 2/6 consts: collected+sorted in {:.1}s, \ streaming put...", sec_start.elapsed().as_secs_f64(), ); @@ -1216,7 +1413,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 2/5 consts done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 2/6 consts done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1225,7 +1422,37 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 3: Names (Address -> Name component, topologically sorted) + // Section 3: anon_hints (Address -> ReducibilityHints) + // + // The canonical hint channel, placed before the metadata sections + // so `get_anon`/`get_anon_mmap` can stop right after it (they + // never touch names/named/comms). `anon_hints` is the single home + // for hints: the compiler registers them per constant address + // (`Env::register_hint`) and readers fill the map from this very + // section. Hints are performance-only advice and intentionally + // NOT covered by the consts merkle root. + // ───────────────────────────────────────────────────────────────────── + let sec_start = std::time::Instant::now(); + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, buf); + for (addr, hints) in &hint_pairs { + put_address(addr, buf); + hints.put_ser(buf); + } + if !quiet { + eprintln!( + "[Env::put] section 3/6 anon_hints done: {} entries in {:.1}s \ + ({} bytes so far)", + hint_pairs.len(), + sec_start.elapsed().as_secs_f64(), + buf.len(), + ); + } + + // ───────────────────────────────────────────────────────────────────── + // Section 4: Names (Address -> Name component, topologically sorted) // ───────────────────────────────────────────────────────────────────── // Topological sort ensures parents come before children so the name // index assigned during serialization is valid for all references that @@ -1234,14 +1461,14 @@ impl Env { let sec_start = std::time::Instant::now(); if !quiet { eprintln!( - "[Env::put] section 3/5 names: topo-sorting {} entries", + "[Env::put] section 4/6 names: topo-sorting {} entries", self.names.len(), ); } let sorted_names = topological_sort_names(&self.names); if !quiet { eprintln!( - "[Env::put] section 3/5 names: topo-sorted in {:.1}s, serializing...", + "[Env::put] section 4/6 names: topo-sorted in {:.1}s, serializing...", sec_start.elapsed().as_secs_f64(), ); } @@ -1255,7 +1482,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 3/5 names done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 4/6 names done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1264,7 +1491,7 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 4: Named (Name -> Named metadata with indexed addresses) + // Section 5: Named (Name -> Named metadata with indexed addresses) // ───────────────────────────────────────────────────────────────────── // Named values are the *largest* per-entry (each carries a ConstantMeta // with metadata arenas), so the streaming pattern's win is greatest @@ -1274,13 +1501,12 @@ impl Env { // single atomic refcount increment (<1s for 733k). let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 4/5 named: {} entries", self.named.len(),); + eprintln!("[Env::put] section 5/6 named: {} entries", self.named.len(),); } let mut named_keys: Vec = self.named.iter().map(|e| e.key().clone()).collect(); - // Sort by the cached name hash bytes (same key used by every existing - // Section 4 ordering guarantee). `par_sort_unstable_by` uses rayon to - // parallelize the compare across all cores. + // Sort by the cached name hash bytes — the section's canonical + // order (ascending name hash, which is exactly `Name`'s `Ord`). #[cfg(not(target_arch = "riscv64"))] named_keys.par_sort_unstable_by(|a, b| { a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) @@ -1289,13 +1515,6 @@ impl Env { named_keys.sort_unstable_by(|a, b| { a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) }); - if !quiet { - eprintln!( - "[Env::put] section 4/5 named: collected+sorted in {:.1}s, \ - streaming put...", - sec_start.elapsed().as_secs_f64(), - ); - } let put_start = std::time::Instant::now(); put_u64(named_keys.len() as u64, buf); for name in &named_keys { @@ -1306,7 +1525,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 4/5 named done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 5/6 named done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1315,11 +1534,11 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 5: Comms (Address -> Comm) — typically empty on compile path + // Section 6: Comms (Address -> Comm) — typically empty on compile path // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 5/5 comms: {} entries", self.comms.len(),); + eprintln!("[Env::put] section 6/6 comms: {} entries", self.comms.len(),); } let mut comm_addrs: Vec
= self.comms.iter().map(|e| e.key().clone()).collect(); @@ -1336,33 +1555,12 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 5/5 comms done in {:.1}s ({} bytes so far)", + "[Env::put] section 6/6 comms done in {:.1}s ({} bytes so far)", sec_start.elapsed().as_secs_f64(), buf.len(), ); } - // Optional trailing section: anon_hints (Address -> ReducibilityHints). - // `get_anon` normally HARVESTS hints from the Named section; a closure - // sub-env (built for shard injection) DROPS that section, which would - // force the kernel to `Regular(0)` and add a def-eq overhead vs whole-env - // proving. Carrying the hints here lets the guest reproduce vanilla kernel - // behavior exactly. Written only when populated, so compiler-produced - // `.ixe` (empty map) are byte-identical to before; loaders read it only if - // bytes remain after comms. Hints are performance-only (the `Regular(0)` - // fallback is always correct), so this section is intentionally NOT covered - // by the consts merkle root. - if !self.anon_hints.is_empty() { - let mut hint_addrs: Vec
= - self.anon_hints.keys().cloned().collect(); - hint_addrs.sort_unstable(); - put_u64(hint_addrs.len() as u64, buf); - for addr in &hint_addrs { - put_address(addr, buf); - self.anon_hints[addr].put_ser(buf); - } - } - if !quiet { eprintln!( "[Env::put] ALL DONE: {} bytes in {:.1}s", @@ -1411,6 +1609,25 @@ impl Env { let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, &mut buf); + // Bundle header fields: distinguished root + assumed-present list + // (mirrors `Env::put`, including the writer-side main sanity check). + if let Some(m) = &self.main + && self.consts.get(m).is_none() + { + return Err(format!( + "Env::put_file: main {} not present in consts", + m.hex() + )); + } + put_opt_addr(&self.main, &mut buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, &mut buf); + for addr in &assumption_addrs { + put_address(addr, &mut buf); + } + // Section 1: Blobs let mut blob_addrs: Vec
= self.blobs.iter().map(|e| e.key().clone()).collect(); @@ -1440,7 +1657,19 @@ impl Env { written += bytes.len() as u64; } } - // Section 3: Names (topologically sorted; builds the name index the + // Section 3: anon_hints — the canonical hint channel, serialized + // straight from the map (see `Env::put`). + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, &mut buf); + for (addr, hints) in &hint_pairs { + put_address(addr, &mut buf); + hints.put_ser(&mut buf); + emit!(); + } + + // Section 4: Names (topologically sorted; builds the name index the // Named section encodes through). let sorted_names = topological_sort_names(&self.names); let mut name_index: NameIndex = NameIndex::new(); @@ -1452,7 +1681,7 @@ impl Env { emit!(); } - // Section 4: Named — the largest per-entry section, and the only + // Section 5: Named — the largest per-entry section, and the only // one whose per-entry encode is CPU-heavy (demoted entries decode // and re-encode through the name index). Chunks encode in parallel // into per-entry buffers and drain to the writer in order; the @@ -1481,7 +1710,7 @@ impl Env { written += b.len() as u64; } } - // Section 5: Comms + // Section 6: Comms let mut comm_addrs: Vec
= self.comms.iter().map(|e| e.key().clone()).collect(); comm_addrs.par_sort_unstable(); @@ -1494,64 +1723,31 @@ impl Env { } } - // Optional trailing anon_hints section — same condition as `put`. - if !self.anon_hints.is_empty() { - let mut hint_addrs: Vec
= - self.anon_hints.keys().cloned().collect(); - hint_addrs.sort_unstable(); - put_u64(hint_addrs.len() as u64, &mut buf); - for addr in &hint_addrs { - put_address(addr, &mut buf); - self.anon_hints[addr].put_ser(&mut buf); - } - } - emit!(); w.flush().map_err(|e| format!("Env::put_file: flush: {e}"))?; Ok(written) } /// Deserialize an Env from bytes. + /// + /// The whole buffer must be consumed — trailing bytes after the + /// final section are rejected (truncation/concatenation guard). + /// `parse_lazy_index` enforces the same; the early-stop readers + /// (`get_anon`, `get_anon_mmap`) cannot make that check by design. pub fn get(buf: &mut &[u8]) -> Result { - // Header - let tag = Tag4::get(buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get: expected Env variant 0, got {}", - tag.size - )); - } - - // Canonical merkle root (fixed 32 bytes). For empty const sets the - // stored value is `zero_address()`. Verified against the - // recomputed value at the end of deserialization. - let stored_root = get_address(buf)?; + // Header: tag + stored merkle root (verified at the end against + // the recomputed root; empty const sets store `zero_address()`) + + // bundle fields. + let (stored_root, main, assumptions) = read_env_header(buf, "Env::get")?; #[cfg_attr(not(target_arch = "riscv64"), allow(unused_mut))] let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs - let num_blobs = get_u64(buf)?; - for _ in 0..num_blobs { - let addr = get_address(buf)?; - let len = get_u64(buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - *buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (hash-verified per entry) + for (addr, bytes) in read_blob_section(buf, "Env::get")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (lazy: read length prefix, slice bytes, defer parse) @@ -1587,7 +1783,19 @@ impl Env { .insert(addr, crate::lazy::LazyConstant::from_bytes(bytes.into())); } - // Section 3: Names (build lookup table and reverse index for metadata) + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!("Env::get: main {} not present in consts", m.hex())); + } + + // Section 3: anon_hints + for (addr, hints) in read_hints_section(buf)? { + env.anon_hints.insert(addr, hints); + } + + // Section 4: Names (build lookup table and reverse index for metadata) let num_names = get_u64(buf)?; let mut names_lookup: FxHashMap = FxHashMap::default(); let mut name_reverse_index: NameReverseIndex = @@ -1606,7 +1814,7 @@ impl Env { env.names.insert(addr, name); } - // Section 4: Named (use indexed deserialization for metadata) + // Section 5: Named (use indexed deserialization for metadata) let num_named = get_u64(buf)?; for _ in 0..num_named { let name_addr = get_address(buf)?; @@ -1617,7 +1825,7 @@ impl Env { env.named.insert(name, named); } - // Section 5: Comms + // Section 6: Comms let num_comms = get_u64(buf)?; for _ in 0..num_comms { let addr = get_address(buf)?; @@ -1625,9 +1833,6 @@ impl Env { env.comms.insert(addr, comm); } - // Optional trailing anon_hints section (see `Env::put`). - read_anon_hints_section(buf, &mut env)?; - // Verify the stored merkle root matches what we'd compute from // env.consts. Empty const set → expected = zero_address(). // Rejects any tampering with the header. @@ -1644,6 +1849,15 @@ impl Env { )); } + // Comms is the final section; anything after it is truncation + // damage or concatenated garbage. + if !buf.is_empty() { + return Err(format!( + "Env::get: {} trailing bytes after final section", + buf.len() + )); + } + Ok(env) } @@ -1657,48 +1871,38 @@ impl Env { /// - the `names` table and each `Named`'s `ExprMetaArena` are parsed only to /// advance the cursor and are then dropped, keeping just `name → addr` and /// the per-`Defn` reducibility hint; - /// - `comms` are skipped entirely. + /// - §3 hints and §6 comms are carried verbatim (both tiny), so an `Env` + /// rebuilt from the index (see [`Env::from_lazy_index`]) matches the full + /// reader on every anon-visible section. /// /// `data` must be the whole buffer (offsets are relative to its start). The - /// env-level merkle root over const addresses is still re-verified. + /// env-level merkle root over const addresses is still re-verified. Since + /// this reader now consumes every section, trailing bytes are rejected + /// exactly as in [`Env::get`]. pub fn parse_lazy_index(data: &[u8]) -> Result { + Self::parse_lazy_index_with_names(data).map(|(index, _)| index) + } + + /// [`Env::parse_lazy_index`], additionally returning the §4 + /// Address→Name component lookup. The lookup is built during the + /// walk regardless (§4 parsing resolves parent references through + /// it); the plain variant just drops it. Consumers that resolve + /// metadata name references from §5 windows — the streaming prune — + /// need it. + pub fn parse_lazy_index_with_names( + data: &[u8], + ) -> Result<(LazyIndex, FxHashMap), String> { let mut buf: &[u8] = data; - // Header: Tag4 (flag/variant) + canonical merkle root. - let tag = Tag4::get(&mut buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "parse_lazy_index: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "parse_lazy_index: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(&mut buf)?; + // Header: tag + merkle root + bundle fields. + let (stored_root, main, assumptions) = + read_env_header(&mut buf, "parse_lazy_index")?; - let mut index = LazyIndex::default(); + let mut index = LazyIndex { main, assumptions, ..LazyIndex::default() }; - // Section 1: Blobs (copied — small, and the kernel ingests their bytes). - let num_blobs = get_u64(&mut buf)?; - for _ in 0..num_blobs { - let addr = get_address(&mut buf)?; - let len = get_u64(&mut buf)? as usize; - if buf.len() < len { - return Err(format!( - "parse_lazy_index: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - buf = rest; - index.blobs.push((addr, bytes.to_vec())); - } + // Section 1: Blobs (copied — small, and the kernel ingests their + // bytes; hash-verified per entry). + index.blobs = read_blob_section(&mut buf, "parse_lazy_index")?; // Section 2: Consts — record offset windows, never parse the bodies. let num_consts = get_u64(&mut buf)?; @@ -1718,8 +1922,14 @@ impl Env { index.consts.push(LazyConstSlice { addr, offset, len }); } - // Section 3: Names — parsed to build the index for metadata decoding, then - // dropped (the check never consults the name component table). + // Section 3: anon_hints — kept verbatim on the index for + // full-reader parity. + index.hints = read_hints_section(&mut buf)?; + + // Section 4: Names — parsed to build the index for metadata + // decoding. The reverse index is retained on the LazyIndex so §5 + // entries can be re-parsed standalone later (the diff meta sweep); + // the Address→Name lookup is dropped (LazyNamed carries the Names). let num_names = get_u64(&mut buf)?; let mut names_lookup: FxHashMap = FxHashMap::default(); let mut name_reverse_index: NameReverseIndex = @@ -1732,24 +1942,41 @@ impl Env { name_reverse_index.push(addr.clone()); names_lookup.insert(addr, name); } + index.name_reverse_index = name_reverse_index; - // Section 4: Named — keep `name → addr` and the Defn reducibility hint; the - // full ConstantMeta (arena, CallSite, ...) is parsed then discarded. + // Section 5: Named — keep `name → addr`; the full ConstantMeta + // (arena, CallSite, ...) is parsed then discarded. The section + // offset is recorded so a [`NamedMetaCursor`] can re-walk it + // without re-parsing the earlier sections. + index.named_section_offset = data.len() - buf.len(); let num_named = get_u64(&mut buf)?; for _ in 0..num_named { let name_addr = get_address(&mut buf)?; - let named = get_named_indexed(&mut buf, &name_reverse_index)?; + let named = get_named_indexed(&mut buf, &index.name_reverse_index)?; let name = names_lookup.get(&name_addr).cloned().ok_or_else(|| { format!("parse_lazy_index: missing name for addr {:?}", name_addr) })?; - let hint = match &named.meta().info { - super::metadata::ConstantMetaInfo::Def { hints, .. } => Some(*hints), - _ => None, - }; - index.named.push(LazyNamed { name, addr: named.addr, hint }); + index.named.push(LazyNamed { name, addr: named.addr }); + } + + // Section 6: Comms — carried verbatim (tiny; empty for + // compile-produced envs). The check path ignores them, but + // `Env::from_lazy_index` consumers (e.g. `ix diff`) want parity + // with the full reader. + let num_comms = get_u64(&mut buf)?; + for _ in 0..num_comms { + let addr = get_address(&mut buf)?; + let comm = Comm::get(&mut buf)?; + index.comms.push((addr, comm)); } - // Section 5 (comms) is skipped: not needed by the check path. + // Every section consumed — enforce EOF like `Env::get`. + if !buf.is_empty() { + return Err(format!( + "parse_lazy_index: {} trailing bytes after final section", + buf.len() + )); + } // Re-verify the merkle root over const addresses (header integrity). let mut const_addrs: Vec
= @@ -1765,63 +1992,104 @@ impl Env { )); } - Ok(index) + // `main` must reference a constant present in the file. + if let Some(m) = &index.main + && const_addrs.binary_search(m).is_err() + { + return Err(format!( + "parse_lazy_index: main {} not present in consts", + m.hex() + )); + } + + Ok((index, names_lookup)) } - /// Anonymous-only deserialization: read the header + blobs + - /// consts sections, parse-and-discard the metadata sections - /// (names / named / comms). + /// [`Env::prune_to_closure`] over a lazily-loaded env — the + /// streaming-metadata pack path. `self` must be the env built by + /// [`Env::from_lazy_index`]/[`Env::from_lazy_index_mmap`] from + /// `index` over `data`, and `names` the §4 lookup from + /// [`Env::parse_lazy_index_with_names`]. Each fixpoint round + /// re-streams §5 with a [`NamedMetaCursor`], materializing `Named` + /// entries only for carried constants — resident metadata is + /// O(survivors) instead of O(env), the full reader's cost. The carry + /// logic is [`Env::carry_named_entry`], shared with the in-memory + /// prune, so the two paths produce identical bundles. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure_streaming( + &self, + index: &LazyIndex, + data: &[u8], + names: &FxHashMap, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; + let mut named_done: FxHashSet = FxHashSet::default(); + loop { + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; + + // ── Named pass, streamed: every §5 entry is parsed (boundaries + // require it — no length sidecar), but only entries whose + // constant was carried this round materialize into `out`. + let mut cursor = NamedMetaCursor::open(data, index)?; + let mut i = 0usize; + while let Some((_, named)) = cursor.next_entry()? { + let name = &index.named[i].name; + i += 1; + if !out.consts.contains_key(&named.addr) || named_done.contains(name) { + continue; + } + named_done.insert(name.clone()); + Self::carry_named_entry( + &mut out, + name, + &named, + &|na| names.get(na).cloned(), + &|ba| self.get_blob(ba), + &mut visited, + &mut pending, + )?; + } + + if pending.is_empty() { + break; + } + } + Ok(out) + } + + /// Anonymous-only deserialization: read the header + §1 blobs + + /// §2 consts + §3 anon_hints, then STOP — the metadata sections + /// (§4 names / §5 named / §6 comms) are laid out after the hints + /// precisely so this reader never has to touch them. /// - /// Returns an `Env` with populated `consts` (lazy) and `blobs`, and - /// **empty** `named` / `names` / `comms`. The merkle-root header is - /// re-verified against the recomputed root over `consts.keys()`, - /// exactly as in [`Env::get`]. + /// Returns an `Env` with populated `consts` (lazy), `blobs`, and + /// `anon_hints`, and **empty** `named` / `names` / `comms`. The + /// merkle-root header is re-verified against the recomputed root + /// over `consts.keys()`, exactly as in [`Env::get`]. Because the + /// cursor stops at §3, trailing-buffer checks are impossible here + /// by design — only the full [`Env::get`] enforces EOF. /// - /// Why "parse and discard"? Sections 3-5 lack a section-level length - /// prefix today (only section 2 has one), so we can't byte-skip - /// them without parsing. Parsing into local scopes that drop on - /// return still wins us the steady-state memory: the returned `Env` - /// is metadata-free, and the temporary lookup tables / parsed - /// metadata values are reclaimed before this function returns. + /// Hints come straight from §3: the writer always emits that + /// section, deriving it from Named metadata when the in-memory map + /// is empty, so the historical read-time harvest from §Named is + /// gone. /// /// Used by the anon-mode kernel path so a verifier holding only - /// content addresses doesn't pay the long-term cost of metadata - /// sections it will never consult. + /// content addresses doesn't pay for metadata it will never consult. pub fn get_anon(buf: &mut &[u8]) -> Result { // Header (same as Env::get) - let tag = Tag4::get(buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get_anon: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get_anon: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(buf)?; + let (stored_root, main, assumptions) = + read_env_header(buf, "Env::get_anon")?; let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs (kept) - let num_blobs = get_u64(buf)?; - for _ in 0..num_blobs { - let addr = get_address(buf)?; - let len = get_u64(buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get_anon: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - *buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (kept; hash-verified per entry) + for (addr, bytes) in read_blob_section(buf, "Env::get_anon")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (kept, lazy) @@ -1851,55 +2119,27 @@ impl Env { .insert(addr, crate::lazy::LazyConstant::from_bytes(bytes.into())); } - // Section 3: Names — parse and DISCARD. We still need a populated - // `names_lookup` and `name_reverse_index` so section 4's indexed - // metadata parses correctly, but both go out of scope before - // returning so the steady-state `Env` carries no name data. - let num_names = get_u64(buf)?; - let mut names_lookup: FxHashMap = FxHashMap::default(); - let mut name_reverse_index: NameReverseIndex = - Vec::with_capacity(num_names as usize + 1); - let anon_addr = Address::from_blake3_hash(*Name::anon().get_hash()); - names_lookup.insert(anon_addr, Name::anon()); - for _ in 0..num_names { - let addr = get_address(buf)?; - let name = get_name_component(buf, &names_lookup)?; - name_reverse_index.push(addr.clone()); - names_lookup.insert(addr, name); - } - - // Section 4: Named — parse and mostly discard, but harvest - // `ReducibilityHints` from each `Def` variant into `env.anon_hints`. - // Hints are performance advice (lazy-delta tiebreak); the kernel's - // anon-mode correctness model is preserved either way. Without - // them, every Definition is forced to `Regular(0)` and the kernel - // can chew through `MAX_WHNF_FUEL` on definitions Lean would have - // marked `Abbrev`/`Regular(h)`. - let num_named = get_u64(buf)?; - for _ in 0..num_named { - let _name_addr = get_address(buf)?; - let named = get_named_indexed(buf, &name_reverse_index)?; - if let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &named.meta().info - { - env.anon_hints.insert(named.addr.clone(), *hints); - } + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!( + "Env::get_anon: main {} not present in consts", + m.hex() + )); } - // Section 5: Comms — parse and DISCARD. - let num_comms = get_u64(buf)?; - for _ in 0..num_comms { - let _addr = get_address(buf)?; - let _comm = Comm::get(buf)?; + // Section 3: anon_hints. Hints are performance advice (lazy-delta + // tiebreak); the kernel's anon-mode correctness model is preserved + // either way. Without them, every Definition is forced to + // `Regular(0)` and the kernel can chew through `MAX_WHNF_FUEL` on + // definitions Lean would have marked `Abbrev`/`Regular(h)`. + for (addr, hints) in read_hints_section(buf)? { + env.anon_hints.insert(addr, hints); } - // Optional trailing anon_hints section (see `Env::put`). For a closure - // sub-env this carries the hints the dropped Named section would have, so - // the kernel reproduces vanilla behavior with no def-eq overhead. - read_anon_hints_section(buf, &mut env)?; - - drop(names_lookup); - drop(name_reverse_index); + // Sections 4-6 (names / named / comms) are laid out after the + // hints precisely so this reader can stop here. // Verify merkle root over loaded consts. let mut const_addrs: Vec
= @@ -1929,11 +2169,10 @@ impl Env { /// the mapping stays alive as long as any consumer holds the env or /// any clone of a `LazyConstant` from it. /// - /// Sections 1 (blobs), 3 (names), 4 (named), and 5 (comms) are - /// handled the same way as `get_anon`: blobs are heap-copied (they - /// are small and consumed eagerly), names and named are - /// parse-and-discard (with hints harvested into `env.anon_hints`), - /// comms are skipped. + /// Sections are handled the same way as `get_anon`: §1 blobs are + /// heap-copied and hash-verified (small, eagerly consumed), §3 + /// anon_hints is read into `env.anon_hints`, and the reader stops + /// there — the metadata sections (§4-§6) are never touched. /// /// On Linux, the kernel's adaptive readahead handles the linear /// scan during section parsing efficiently; subsequent random @@ -1990,39 +2229,17 @@ impl Env { let mut buf: &[u8] = mmap_full; // Header (same shape as Env::get_anon) - let tag = Tag4::get(&mut buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get_anon_mmap: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get_anon_mmap: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(&mut buf)?; + let (stored_root, main, assumptions) = + read_env_header(&mut buf, "Env::get_anon_mmap")?; let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs (heap-copied; small, eagerly consumed) - let num_blobs = get_u64(&mut buf)?; - for _ in 0..num_blobs { - let addr = get_address(&mut buf)?; - let len = get_u64(&mut buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get_anon_mmap: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (heap-copied; small, eagerly consumed; + // hash-verified per entry) + for (addr, bytes) in read_blob_section(&mut buf, "Env::get_anon_mmap")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (mmap-backed lazy windows) @@ -2060,47 +2277,23 @@ impl Env { buf = &buf[len..]; } - // Section 3: Names — parse and DISCARD (needed transiently so - // section 4's indexed metadata can be decoded). - let num_names = get_u64(&mut buf)?; - let mut names_lookup: FxHashMap = FxHashMap::default(); - let mut name_reverse_index: NameReverseIndex = - Vec::with_capacity(num_names as usize + 1); - let anon_addr = Address::from_blake3_hash(*Name::anon().get_hash()); - names_lookup.insert(anon_addr, Name::anon()); - for _ in 0..num_names { - let addr = get_address(&mut buf)?; - let name = get_name_component(&mut buf, &names_lookup)?; - name_reverse_index.push(addr.clone()); - names_lookup.insert(addr, name); - } - - // Section 4: Named — harvest `ReducibilityHints` from `Def` - // entries into `env.anon_hints`; discard the rest. See `get_anon` - // for the rationale. - let num_named = get_u64(&mut buf)?; - for _ in 0..num_named { - let _name_addr = get_address(&mut buf)?; - let named = get_named_indexed(&mut buf, &name_reverse_index)?; - if let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &named.meta().info - { - env.anon_hints.insert(named.addr.clone(), *hints); - } + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!( + "Env::get_anon_mmap: main {} not present in consts", + m.hex() + )); } - // Section 5: Comms — parse and DISCARD. - let num_comms = get_u64(&mut buf)?; - for _ in 0..num_comms { - let _addr = get_address(&mut buf)?; - let _comm = Comm::get(&mut buf)?; + // Section 3: anon_hints — see `get_anon` for the rationale. + for (addr, hints) in read_hints_section(&mut buf)? { + env.anon_hints.insert(addr, hints); } - // Optional trailing anon_hints section (see `Env::put`). - read_anon_hints_section(&mut buf, &mut env)?; - - drop(names_lookup); - drop(name_reverse_index); + // Sections 4-6 (names / named / comms) are laid out after the + // hints precisely so this reader can stop here. // Verify merkle root over loaded consts (same as get_anon). let mut const_addrs: Vec
= @@ -2126,20 +2319,29 @@ impl Env { Ok(buf.len()) } - /// Calculate serialized size with breakdown by section. + /// Calculate serialized size with breakdown by section: + /// `(header, blobs, consts, anon_hints, names, named, comms)`. pub fn serialized_size_breakdown( &self, - ) -> Result<(usize, usize, usize, usize, usize, usize), String> { + ) -> Result<(usize, usize, usize, usize, usize, usize, usize), String> { let mut buf = Vec::new(); - // Header + merkle root (matches Env::put layout; root is always - // 32 bytes, with `zero_address()` as the empty-env sentinel). + // Header: tag + merkle root (32 bytes, `zero_address()` sentinel + // for empty const sets) + bundle fields (matches Env::put layout). Tag4::new(Self::FLAG, 0).put(&mut buf); let mut const_addrs: Vec
= self.consts.iter().map(|e| e.key().clone()).collect(); const_addrs.sort_unstable(); let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, &mut buf); + put_opt_addr(&self.main, &mut buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, &mut buf); + for addr in &assumption_addrs { + put_address(addr, &mut buf); + } let header_size = buf.len(); // Section 1: Blobs @@ -2162,7 +2364,19 @@ impl Env { } let consts_size = buf.len() - before_consts; - // Section 3: Names (also build name index) + // Section 3: anon_hints (serialized straight from the map) + let before_hints = buf.len(); + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, &mut buf); + for (addr, hints) in &hint_pairs { + put_address(addr, &mut buf); + hints.put_ser(&mut buf); + } + let hints_size = buf.len() - before_hints; + + // Section 4: Names (also build name index) let before_names = buf.len(); let sorted_names = topological_sort_names(&self.names); let mut name_index: NameIndex = NameIndex::new(); @@ -2174,7 +2388,7 @@ impl Env { } let names_size = buf.len() - before_names; - // Section 4: Named (use indexed serialization) + // Section 5: Named (use indexed serialization) let before_named = buf.len(); put_u64(self.named.len() as u64, &mut buf); for entry in self.named.iter() { @@ -2183,7 +2397,7 @@ impl Env { } let named_size = buf.len() - before_named; - // Section 5: Comms + // Section 6: Comms let before_comms = buf.len(); put_u64(self.comms.len() as u64, &mut buf); for entry in self.comms.iter() { @@ -2196,6 +2410,7 @@ impl Env { header_size, blobs_size, consts_size, + hints_size, names_size, named_size, comms_size, @@ -2385,7 +2600,7 @@ mod tests { } fn gen_env(g: &mut Gen) -> Env { - let env = Env::new(); + let mut env = Env::new(); // Generate blobs let num_blobs = gen_range(g, 0..10); @@ -2407,12 +2622,14 @@ mod tests { // Generate constants and named entries let num_consts = gen_range(g, 0..10); + let mut const_addrs: Vec
= Vec::new(); for i in 0..num_consts { let constant = gen_constant(g); let mut buf = Vec::new(); constant.put(&mut buf); let addr = Address::hash(&buf); env.store_const(addr.clone(), constant); + const_addrs.push(addr.clone()); // Create a named entry for this constant if !names.is_empty() { @@ -2448,6 +2665,31 @@ mod tests { env.comms.insert(addr, comm); } + // Bundle fields. `main` must reference a stored const — `Env::put` + // validates that; `assumptions` are opaque addresses (no content + // behind them is required), so random ones exercise the encoding. + if !const_addrs.is_empty() && bool::arbitrary(g) { + let idx = usize::arbitrary(g) % const_addrs.len(); + env.main = Some(const_addrs[idx].clone()); + } + let num_assumptions = gen_range(g, 0..5); + for _ in 0..num_assumptions { + env.assumptions.insert(Address::arbitrary(g)); + } + + // anon_hints (keyed by arbitrary addresses — the map is advisory + // and the hints section is serialized straight from it). + let num_hints = gen_range(g, 0..4); + for _ in 0..num_hints { + let variant: u8 = Arbitrary::arbitrary(g); + let hint = match variant % 3 { + 0 => ReducibilityHints::Opaque, + 1 => ReducibilityHints::Abbrev, + _ => ReducibilityHints::Regular(Arbitrary::arbitrary(g)), + }; + env.anon_hints.insert(Address::arbitrary(g), hint); + } + env } @@ -2546,6 +2788,33 @@ mod tests { } } + // Bundle fields + hints (gen_env metas are all Empty, so §3 + // derivation contributes nothing and plain equality holds). + if env.main != recovered.main { + eprintln!("main mismatch: {:?} vs {:?}", env.main, recovered.main); + return false; + } + if env.assumptions != recovered.assumptions { + eprintln!( + "assumptions mismatch: {} vs {} entries", + env.assumptions.len(), + recovered.assumptions.len() + ); + return false; + } + let hints_match = env.anon_hints.len() == recovered.anon_hints.len() + && env.anon_hints.iter().all(|e| { + recovered.anon_hints.get(e.key()).map(|r| *r) == Some(*e.value()) + }); + if !hints_match { + eprintln!( + "anon_hints mismatch: {} vs {} entries", + env.anon_hints.len(), + recovered.anon_hints.len() + ); + return false; + } + true }, Err(e) => { @@ -2683,21 +2952,39 @@ mod tests { assert!(res.is_err(), "tampered root should be rejected"); } + /// Byte offset of the first constant's payload within a serialized + /// env buffer, computed by walking the header + §1 (blobs) + the §2 + /// prefix with the real parsers — so tampering tests track layout + /// changes instead of hardcoding offsets. + fn first_const_payload_offset(buf: &[u8]) -> usize { + let mut cur: &[u8] = buf; + read_env_header(&mut cur, "test").unwrap(); + read_blob_section(&mut cur, "test").unwrap(); + let n = get_u64(&mut cur).unwrap(); + assert!(n >= 1, "need at least one const to locate"); + let _addr = get_address(&mut cur).unwrap(); + let _len = Tag0::get(&mut cur).unwrap(); + buf.len() - cur.len() + } + + /// Byte offset of the first blob's payload (same technique as + /// [`first_const_payload_offset`]). + fn first_blob_payload_offset(buf: &[u8]) -> usize { + let mut cur: &[u8] = buf; + read_env_header(&mut cur, "test").unwrap(); + let n = get_u64(&mut cur).unwrap(); + assert!(n >= 1, "need at least one blob to locate"); + let _addr = get_address(&mut cur).unwrap(); + let _len = get_u64(&mut cur).unwrap(); + buf.len() - cur.len() + } + /// Flip a byte inside the first const's payload bytes (not its /// stored address): merkle still validates over `consts.keys()`, so /// the per-entry `Address::hash(bytes) == addr` check is what must /// reject this corruption. Without that check, `Env::get` would /// succeed and the failure would surface much later inside /// `LazyConstant::get` with a misleading parse error. - /// - /// Header layout for an env with empty blobs and one const: - /// [0] Tag4 (0xE0) - /// [1..33) merkle root (32 bytes) - /// [33] Section 1 (blobs) count = 0 (Tag0) - /// [34] Section 2 (consts) count = 1 (Tag0) - /// [35..67) const address (32 bytes) - /// [67] Tag0 length of const bytes - /// [68..] const bytes (target for tampering) #[test] fn env_const_bytes_tampering_rejected_by_get() { let env = Env::new(); @@ -2705,7 +2992,7 @@ mod tests { let mut buf = Vec::new(); env.put(&mut buf).unwrap(); // Flip a byte well inside the const payload. - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len(), "expected const bytes at offset {off}"); buf[off] ^= 0xFF; let res = Env::get(&mut buf.as_slice()); @@ -2722,7 +3009,7 @@ mod tests { env.store_const(Address::hash(b"a"), defn_const(vec![])); let mut buf = Vec::new(); env.put(&mut buf).unwrap(); - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len()); buf[off] ^= 0xFF; let res = Env::get_anon(&mut buf.as_slice()); @@ -2740,7 +3027,7 @@ mod tests { env.store_const(Address::hash(b"a"), defn_const(vec![])); let mut buf = Vec::new(); env.put(&mut buf).unwrap(); - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len()); buf[off] ^= 0xFF; // mmap requires a real file @@ -2885,4 +3172,185 @@ mod tests { assert_eq!(pre_a, post_a, "`a` content should be stable across unlink"); assert_ne!(post_a, post_b, "discriminators should still differentiate"); } + + // --------------------------------------------------------------------------- + // Bundle header fields (main / assumptions) + §1/§3 integrity + // --------------------------------------------------------------------------- + + #[test] + fn env_main_and_assumptions_roundtrip_all_readers() { + let mut env = Env::new(); + let a = store_canonical(&env, defn_const(vec![])); + store_canonical(&env, defn_const_discriminator(vec![], 1)); + env.main = Some(a.clone()); + env.assumptions.insert(Address::hash(b"assume-1")); + env.assumptions.insert(Address::hash(b"assume-2")); + + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + + let full = Env::get(&mut buf.as_slice()).unwrap(); + assert_eq!(full.main, Some(a.clone())); + assert_eq!(full.assumptions, env.assumptions); + + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!(anon.main, Some(a.clone())); + assert_eq!(anon.assumptions, env.assumptions); + + let index = Env::parse_lazy_index(&buf).unwrap(); + assert_eq!(index.main, Some(a)); + let mut sorted: Vec
= env.assumptions.iter().cloned().collect(); + sorted.sort_unstable(); + assert_eq!(index.assumptions, sorted, "LazyIndex keeps header order"); + } + + #[test] + fn env_put_rejects_main_not_in_consts() { + let mut env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.main = Some(Address::hash(b"not-a-const")); + let mut buf = Vec::new(); + let err = env + .put(&mut buf) + .expect_err("main outside consts must be rejected at write time"); + assert!(err.contains("main"), "got: {err}"); + } + + /// Flip a byte of the serialized `main` address: the header parses, + /// but the resulting address is (w.h.p.) not a stored constant, so + /// the `main ∈ consts` reader check must reject the file. This test + /// intentionally knows the fixed header prefix (tag byte + 32-byte + /// root — same knowledge as `parse_stored_root`). + #[test] + fn env_get_rejects_tampered_main() { + let mut env = Env::new(); + let a = store_canonical(&env, defn_const(vec![])); + env.main = Some(a); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + assert_eq!(buf[33], 0x01, "main opt-flag should be Some"); + buf[34] ^= 0xFF; + let err = Env::get(&mut buf.as_slice()) + .expect_err("tampered main must be rejected"); + assert!(err.contains("main"), "got: {err}"); + } + + #[test] + fn env_get_rejects_unsorted_assumptions() { + let mut env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.assumptions.insert(Address::hash(b"x")); + env.assumptions.insert(Address::hash(b"y")); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + // Locate the header end with the real parser, then swap the two + // 32-byte assumption addresses in place (the writer sorts them). + let header_len = { + let mut cur: &[u8] = &buf; + read_env_header(&mut cur, "test").unwrap(); + buf.len() - cur.len() + }; + let (lo, hi) = (header_len - 64, header_len - 32); + let first: Vec = buf[lo..hi].to_vec(); + let second: Vec = buf[hi..header_len].to_vec(); + buf[lo..hi].copy_from_slice(&second); + buf[hi..header_len].copy_from_slice(&first); + let err = Env::get(&mut buf.as_slice()) + .expect_err("descending assumptions must be rejected"); + assert!(err.contains("strictly ascending"), "got: {err}"); + } + + #[test] + fn env_blob_tampering_rejected_by_all_readers() { + use std::io::Write; + let env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.store_blob(b"blob payload".to_vec()); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + let off = first_blob_payload_offset(&buf) + 2; + buf[off] ^= 0xFF; + + for (reader, res) in [ + ("get", Env::get(&mut buf.as_slice()).err()), + ("get_anon", Env::get_anon(&mut buf.as_slice()).err()), + ("parse_lazy_index", Env::parse_lazy_index(&buf).err()), + ] { + let err = + res.unwrap_or_else(|| panic!("{reader}: tampered blob accepted")); + assert!( + err.contains("blob at idx"), + "{reader}: expected blob verify error, got: {err}" + ); + } + + let tmp = std::env::temp_dir().join("ix_env_blob_tamper_mmap_test.ixe"); + { + let mut f = std::fs::File::create(&tmp).unwrap(); + f.write_all(&buf).unwrap(); + } + let err = Env::get_anon_mmap(&tmp) + .expect_err("get_anon_mmap: tampered blob accepted"); + assert!(err.contains("blob at idx"), "got: {err}"); + std::fs::remove_file(&tmp).ok(); + } + + #[test] + fn env_get_rejects_trailing_garbage_but_get_anon_stops_early() { + let env = Env::new(); + store_canonical(&env, defn_const(vec![])); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + buf.extend_from_slice(&[0xAB, 0xCD, 0xEF]); + let err = Env::get(&mut buf.as_slice()) + .expect_err("trailing bytes must be rejected by the full reader"); + assert!(err.contains("trailing"), "got: {err}"); + // get_anon stops after §3 and never sees the tail — by design. + Env::get_anon(&mut buf.as_slice()) + .expect("get_anon early-stops before the garbage"); + } + + /// §3 is the canonical hint channel: with an empty `anon_hints` map + /// the writer derives the section from Named `Def` metadata, and an + /// env carrying the same hints explicitly serializes byte-identically. + /// The hints section round-trips through both the anon and full + /// readers, and `register_hint`'s merge is order-independent on + /// address collisions. + #[test] + fn env_hints_roundtrip_and_merge_deterministic() { + let env = Env::new(); + let const_addr = store_canonical(&env, defn_const(vec![])); + env.register_hint(const_addr.clone(), ReducibilityHints::Regular(7)); + + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!( + anon.anon_hints.get(&const_addr).map(|r| *r), + Some(ReducibilityHints::Regular(7)) + ); + let full = Env::get(&mut buf.as_slice()).unwrap(); + assert_eq!( + full.anon_hints.get(&const_addr).map(|r| *r), + Some(ReducibilityHints::Regular(7)) + ); + + // Alias collision: registration order must not affect the winner. + let a = Env::new(); + let ca = store_canonical(&a, defn_const(vec![])); + a.register_hint(ca.clone(), ReducibilityHints::Regular(9)); + a.register_hint(ca.clone(), ReducibilityHints::Abbrev); + let b = Env::new(); + let cb = store_canonical(&b, defn_const(vec![])); + b.register_hint(cb.clone(), ReducibilityHints::Abbrev); + b.register_hint(cb.clone(), ReducibilityHints::Regular(9)); + assert_eq!( + a.anon_hints.get(&ca).map(|r| *r), + b.anon_hints.get(&cb).map(|r| *r) + ); + assert_eq!( + a.anon_hints.get(&ca).map(|r| *r), + Some(ReducibilityHints::Abbrev) + ); + } } diff --git a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs index ead830954..cda5885ca 100644 --- a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs +++ b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs @@ -191,7 +191,7 @@ fn add_entries_parallel( // Hints come from env.anon_hints (sidecar). Collect per chunk. for addr in chunk { if let Some(h) = env.anon_hints.get(addr) { - p.hints.push((addr_key(addr), hint_to_g(h))); + p.hints.push((addr_key(addr), hint_to_g(&h))); } } p diff --git a/crates/ixvm-codegen/src/env_handle.rs b/crates/ixvm-codegen/src/env_handle.rs index f2c0354e0..1e1f6e3d8 100644 --- a/crates/ixvm-codegen/src/env_handle.rs +++ b/crates/ixvm-codegen/src/env_handle.rs @@ -6,12 +6,12 @@ //! per-shard FFI calls take a `&EnvHandle` (shared across the batch) so //! the env is parsed exactly once instead of every call. //! -//! `anon_hints` (`Defn` reducibility hints) are populated at handle -//! construction time. `Env::get_anon_mmap` already harvests them; -//! `Env::get` (full-form decode used by the bytes-blob path) does -//! not, so `from_bytes` runs the harvest post-decode. +//! `anon_hints` (`Defn` reducibility hints) come from the `.ixe` §3 +//! hints section, which every writer emits (deriving it from Named +//! metadata when the in-memory map is empty) — both readers used here +//! populate the map directly, so no post-decode harvest is needed. -use ixon::{Env, metadata::ConstantMetaInfo}; +use ixon::Env; pub struct EnvHandle { pub env: Env, @@ -19,38 +19,17 @@ pub struct EnvHandle { impl EnvHandle { /// Load via `Env::get_anon_mmap` (zero-copy mmap of the `.ixe` file). - /// Anon-mode parser already harvests `anon_hints` — no post-pass. pub fn from_ixe_path(path: &std::path::Path) -> Result { let env = Env::get_anon_mmap(path)?; Ok(Self { env }) } /// Decode a serialized env blob (`Ixon.serEnv` output) via - /// `Env::get`, then harvest `anon_hints` from each `Def` named - /// entry. Used by the compiled-Lean-env path where the env is built - /// in Lean memory and serialized for the cross-FFI handoff. + /// `Env::get`. Used by the compiled-Lean-env path where the env is + /// built in Lean memory and serialized for the cross-FFI handoff. pub fn from_bytes(bytes: &[u8]) -> Result { let mut cursor: &[u8] = bytes; - let mut env = Env::get(&mut cursor)?; - // `Env::get` reads named entries but doesn't populate - // `env.anon_hints`. Walk `env.named` and insert each `Def` - // variant's `hints`. Mirrors the in-line harvest inside - // `Env::get_anon`. - let hints: Vec<_> = env - .named - .iter() - .filter_map(|entry| { - let named = entry.value(); - if let ConstantMetaInfo::Def { hints, .. } = &named.meta().info { - Some((named.addr.clone(), *hints)) - } else { - None - } - }) - .collect(); - for (addr, h) in hints { - env.anon_hints.insert(addr, h); - } + let env = Env::get(&mut cursor)?; Ok(Self { env }) } } diff --git a/crates/kernel/src/anon_work.rs b/crates/kernel/src/anon_work.rs index 1f5be74dd..138aacbc1 100644 --- a/crates/kernel/src/anon_work.rs +++ b/crates/kernel/src/anon_work.rs @@ -193,70 +193,11 @@ pub fn closure_addrs( source: &IxonEnv, roots: &[Address], ) -> std::collections::HashSet
{ - use std::collections::{HashSet, VecDeque}; - - use ixon::constant::{ConstantInfo as CI, MutConst as MC}; - - let proj_block = |info: &CI| -> Option
{ - match info { - CI::IPrj(p) => Some(p.block.clone()), - CI::CPrj(p) => Some(p.block.clone()), - CI::RPrj(p) => Some(p.block.clone()), - CI::DPrj(p) => Some(p.block.clone()), - _ => None, - } - }; - - let mut closure: HashSet
= HashSet::default(); - let mut queue: VecDeque
= VecDeque::new(); - let push = - |closure: &mut HashSet
, q: &mut VecDeque
, a: Address| { - if closure.insert(a.clone()) { - q.push_back(a); - } - }; - for r in roots { - push(&mut closure, &mut queue, r.clone()); - } - while let Some(addr) = queue.pop_front() { - if let Some(c) = source.get_const(&addr) { - // 1. Expr-level refs. - for r in &c.refs { - push(&mut closure, &mut queue, r.clone()); - } - // 2. A projection → its Muts block (structural, not in `refs`). - if let Some(b) = proj_block(&c.info) { - push(&mut closure, &mut queue, b); - } - // 3. A Muts block → ALL its member + constructor projection entries. - // Ingressing a block (`ingress_anon_block`) computes these projection - // addresses and requires them present, even when nothing references them - // directly via `refs`. Mirror `build_anon_work`'s enumeration so the - // sub-env carries them (else the guest fails with "computed CPrj address - // … not present in env"). - if let CI::Muts(members) = &c.info { - for (i, m) in members.iter().enumerate() { - let i = i as u64; - let member_addr = match m { - MC::Defn(_) => anon_defn_proj_addr(&addr, i), - MC::Indc(_) => anon_indc_proj_addr(&addr, i), - MC::Recr(_) => anon_recr_proj_addr(&addr, i), - }; - push(&mut closure, &mut queue, member_addr); - if let MC::Indc(ind) = m { - for cidx in 0..ind.ctors.len() as u64 { - push( - &mut closure, - &mut queue, - anon_ctor_proj_addr(&addr, i, cidx), - ); - } - } - } - } - } - } - closure + // The traversal lives in `ixon` (`Env::bfs_closure`) so bundle + // pruning/validation and the shard sub-env share one edge + // definition; this wrapper keeps the kernel-side std-HashSet + // signature. + source.bfs_closure(roots).into_iter().collect() } /// Build a closure sub-env: serialize only the BFS dependency closure of @@ -265,9 +206,9 @@ pub fn closure_addrs( /// per-const integrity check (`hash(bytes) == addr`) and the env merkle root /// still hold. The guest decodes this instead of the whole env, so it pays only /// its closure's decode — essential for envs that don't fit the guest whole -/// (Init, 184 MB doesn't fit the 512 MB Zisk guest). Empty `anon_hints` (no -/// metadata section) is performance-only: ingress falls back to `Regular(0)`, -/// so the typecheck result — and thus the committed claim — is unchanged. +/// (Init, 184 MB doesn't fit the 512 MB Zisk guest). Missing hints are +/// performance-only: ingress falls back to `Regular(0)`, so the typecheck +/// result — and thus the committed claim — is unchanged. /// External refs (not in `source`) are omitted and remain open assumptions, /// exactly as in whole-env. /// @@ -291,7 +232,7 @@ pub fn build_sub_env( #[cfg(not(target_arch = "riscv64"))] fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { let closure = closure_addrs(source, roots); - let mut sub = IxonEnv::new(); + let sub = IxonEnv::new(); for addr in &closure { if let Some(bytes) = source.get_const_bytes(addr) { sub.store_const_lazy(addr.clone(), bytes); @@ -300,9 +241,8 @@ fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { } // else: external ref absent from `source` — omit; stays an open assumption. // Carry the constant's reducibility hint so the guest reproduces vanilla - // kernel behavior. `get_anon` normally harvests hints from the Named - // section, which this sub-env drops; without them the kernel forces - // `Regular(0)` and does extra def-eq work (the ~30% check overhead). + // kernel behavior; without hints the kernel forces `Regular(0)` and does + // extra def-eq work (the ~30% check overhead). if let Some(h) = source.anon_hints.get(addr) { sub.anon_hints.insert(addr.clone(), *h); } diff --git a/crates/kernel/src/ingress.rs b/crates/kernel/src/ingress.rs index b7349bd0b..7d7ad8fe1 100644 --- a/crates/kernel/src/ingress.rs +++ b/crates/kernel/src/ingress.rs @@ -1396,45 +1396,30 @@ fn ingress_defn( // projection addresses; passing `Some(_)` skips the metadata-derived // `build_mut_ctx` call. Meta callers pass `None`. mut_ctx_override: Option>>, - // Anon callers may supply `Some(hints)` to override the default - // `Regular(0)` fall-through when the .ixe carries `Env::anon_hints` - // (harvested by `Env::get_anon` from the otherwise-discarded Named - // metadata). Meta callers pass `None` and pull hints from - // `meta.info` like usual. The override only takes effect when - // `meta.info` is not `Def` (i.e. anon path with empty meta). - hints_override: Option, ) -> Result, KConst)>, String> { let mut cache: ExprCache = FxHashMap::default(); let mut univ_cache: UnivCache = FxHashMap::default(); - let (level_params, arena, type_root, value_root, hints, safety, all_addrs) = - match &meta.info { - ConstantMetaInfo::Def { - lvls, - arena, - type_root, - value_root, - hints, - all, - .. - } => ( - resolve_level_params(lvls, names), - arena, - *type_root, - *value_root, - *hints, - def.safety, - all.clone(), - ), - _ => ( - vec![], - &DEFAULT_ARENA, - 0, - 0, - hints_override.unwrap_or(ReducibilityHints::Regular(0)), - def.safety, - vec![], - ), - }; + // Hints live only in `Env::anon_hints`, keyed by the constant's own + // address, in both anon and meta modes. Without an entry the kernel + // falls back to `Regular(0)` — always correct, just more def-eq work. + let hints = ixon_env + .anon_hints + .get(&self_id.addr) + .map_or(ReducibilityHints::Regular(0), |r| *r); + let safety = def.safety; + let (level_params, arena, type_root, value_root, all_addrs) = match &meta.info + { + ConstantMetaInfo::Def { + lvls, arena, type_root, value_root, all, .. + } => ( + resolve_level_params(lvls, names), + arena, + *type_root, + *value_root, + all.clone(), + ), + _ => (vec![], &DEFAULT_ARENA, 0, 0, vec![]), + }; let mut_ctx = match mut_ctx_override { Some(m) => m, @@ -1656,7 +1641,6 @@ fn ingress_standalone( intern, stats, None, - None, ), IxonCI::Axio(ax) => { @@ -2052,7 +2036,6 @@ fn ingress_muts_block( intern, stats, None, - None, )?); }, } @@ -3683,12 +3666,22 @@ fn drop_ixon_env(_ixon_env: IxonEnv, _quiet: bool) { #[cfg(not(target_arch = "riscv64"))] fn drop_ixon_env(ixon_env: IxonEnv, quiet: bool) { let total_start = Instant::now(); - // `anon_hints` is a small FxHashMap (one entry per Def from the .ixe's - // Named metadata); dropping it inline alongside the bookkeeping below - // is negligible compared to the DashMap dropdance. + // `anon_hints` is small (one entry per Def from the .ixe's hints + // section); `main`/`assumptions` are a single address and a small + // set. Dropping them inline alongside the bookkeeping below is + // negligible compared to the DashMap dropdance. // `..` covers the env's private fields. - let IxonEnv { consts, named, blobs, names, comms, anon_hints: _, .. } = - ixon_env; + let IxonEnv { + consts, + named, + blobs, + names, + comms, + anon_hints: _, + main: _, + assumptions: _, + .. + } = ixon_env; let consts_len = consts.len(); let named_len = named.len(); let names_len = names.len(); @@ -4236,7 +4229,6 @@ fn ingress_anon_standalone( let empty_n2a: FxHashMap = FxHashMap::default(); let mut convert_stats = ConvertStats::new(false); let self_id: KId = KId::new(addr.clone(), ()); - let hints_override = anon_env.anon_hints.get(addr).copied(); let entries = match &constant.info { IxonCI::Defn(def) => ingress_defn::( @@ -4253,7 +4245,6 @@ fn ingress_anon_standalone( &mut kenv.intern, &mut convert_stats, Some(vec![self_id.clone()]), - hints_override, )?, IxonCI::Recr(rec) => ingress_recursor::( rec, @@ -4464,7 +4455,6 @@ pub fn ingress_anon_block( )?; let self_id = KId::::new(proj_addr.clone(), ()); member_kids.push(self_id.clone()); - let hints_override = anon_env.anon_hints.get(&proj_addr).copied(); let entries = ingress_defn::( def, @@ -4480,7 +4470,6 @@ pub fn ingress_anon_block( &mut kenv.intern, &mut convert_stats, Some(mut_ctx.clone()), - hints_override, )?; all_entries.extend(entries); }, diff --git a/docs/Ixon.md b/docs/Ixon.md index f8a2798e8..558e4df4a 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -34,7 +34,7 @@ Every `Constant` in Ixon is serialized and hashed with blake3. The resulting 256 The Ixon format separates: - **Alpha-invariant data** (`Constant`): The mathematical content, hashed for addressing -- **Metadata** (`ConstantMeta`, `ExprMeta`): Names, binder info, reducibility hints—stored separately +- **Metadata** (`ConstantMeta`, `ExprMeta`): Names and binder info—stored separately (reducibility hints live at the environment level, in `Env::anon_hints`) This separation means cosmetic changes (renaming variables) don't change the constant's address. @@ -690,7 +690,7 @@ Per-constant metadata. Each variant stores a name, universe parameter names, an ```rust pub enum ConstantMeta { Empty, // tag 255 - Def { name, lvls, hints, all, ctx, + Def { name, lvls, all, ctx, arena, type_root, value_root }, // tag 0 Axio { name, lvls, arena, type_root }, // tag 1 Quot { name, lvls, arena, type_root }, // tag 2 @@ -706,7 +706,7 @@ pub enum ConstantMeta { | Tag | Variant | Payload | |-----|---------|---------| -| 0 | Def | name_idx, lvl_idxs, hints, all_idxs, ctx_idxs, arena, type_root, value_root | +| 0 | Def | name_idx, lvl_idxs, all_idxs, ctx_idxs, arena, type_root, value_root | | 1 | Axio | name_idx, lvl_idxs, arena, type_root | | 2 | Quot | name_idx, lvl_idxs, arena, type_root | | 3 | Indc | name_idx, lvl_idxs, ctor_idxs, all_idxs, ctx_idxs, arena, type_root | @@ -802,15 +802,22 @@ plus `.ixe`. See `src/ix/ixon/serialize.rs::Env::put` for the byte-level layout. The .ixe layout is a Tag4(0xE, 0) header byte followed by a 32-byte -canonical merkle root and then 5 sections: +canonical merkle root, the bundle header fields, and then 6 sections +(hot data first, metadata last): ``` -Header: Tag4 { flag: 0xE, size: 0 } -- one byte (0xE0) -Root: 32 bytes -- canonical merkle root over - consts.keys(); for empty - const sets this is the - fixed `zero_address` - sentinel. +Header: Tag4 { flag: 0xE, size: 0 } -- one byte (0xE0) +Root: 32 bytes -- canonical merkle root over + consts.keys(); for empty + const sets this is the + fixed `zero_address` + sentinel. +Main: 1 byte (0x00 | 0x01) + 32 bytes if 0x01 + -- optional bundle root; see + "Bundles" below. +Assumptions: count (Tag0) + [Address (32 bytes)]* + -- strictly ascending; the + bundle trust boundary. ``` The root is mandatory (non-optional): every env has a unique canonical @@ -819,12 +826,23 @@ produce byte-identical roots regardless of construction order. Deserialization recomputes the root from `consts` and rejects any mismatch as tampered. +`main` and `assumptions` are NOT covered by the consts merkle root — +`main` is a convenience pointer (readers verify `main ∈ consts`; +consumers holding an externally-expected address must compare), and +the assumptions root, when a claim needs it, is recomputed as +`merkle_root_canonical(leaves)` from the section (identical to the +root `AssumptionTree::canonical` produces over the same leaves). + **Section 1: Blobs** (Address → raw bytes) ``` count (Tag0) [Address (32 bytes) + len (Tag0) + bytes]* ``` +Every reader verifies `blake3(bytes) == addr` per blob entry — a +swapped blob would otherwise silently change a Nat/String literal's +value under an otherwise-valid file. + **Section 2: Constants** (Address → length-prefixed Constant bytes) ``` count (Tag0) @@ -833,42 +851,136 @@ count (Tag0) The Tag0 length sidecar is a **section-level** framing byte: it is not part of the constant's content-addressed bytes. The address is -computed as `blake3` over only the Tag4 constant body. This layout -lets a lazy loader slice each constant directly into a -[`LazyConstant`](../src/ix/ixon/lazy.rs) without parsing its Tag4 -envelope, deferring full deserialization until first access. The -materialized `Constant` is cached so subsequent accesses are free. - -### Anonymous-only loading +computed as `blake3` over only the Tag4 constant body (verified per +entry on load). This layout lets a lazy loader slice each constant +directly into a [`LazyConstant`](../crates/ixon/src/lazy.rs) without +parsing its Tag4 envelope, deferring full deserialization until first +access. -`Env::get_anon` (`src/ix/ixon/serialize.rs`) is a sibling of -`Env::get` that loads only the anonymous sections — header, blobs, -consts — and parses-and-drops the metadata sections (names, named, -comms). The returned `Env` has empty `named`/`names`/`comms` and is -suitable for anon-mode kernel workflows that never consult metadata. -Steady-state memory for a Mathlib-scale env drops from ~3-4 GB -(structured + metadata) to ~1 GB (lazy bytes only). +**Section 3: Reducibility hints** (Address → ReducibilityHints) +``` +count (Tag0) +[Address (32 bytes) + ReducibilityHints]* +``` -Exposed to Lean via `rs_de_env_anon` (`Ix.Ixon.rsDeEnvAnon`). +The canonical (and only) hint channel, keyed by constant address and +sorted ascending. Hints never appear in `ConstantMeta`: the compiler +registers each definition's hints into `Env::anon_hints` +(`Env::register_hint`, an order-independent merge on alias +collisions), writers serialize that map here, and readers load it +back. Hints are performance-only advice (the `Regular(0)` fallback is +always correct) and are intentionally outside the consts merkle root. -**Section 3: Names** (Address → NameComponent, topologically sorted) +**Section 4: Names** (Address → NameComponent, topologically sorted) ``` count (Tag0) [Address (32 bytes) + NameComponent]* ``` -**Section 4: Named** (Name Address → Named with indexed metadata) +**Section 5: Named** (Name Address → Named with indexed metadata) ``` count (Tag0) [NameAddress (32 bytes) + ConstAddress (32 bytes) + ConstantMeta]* ``` -**Section 5: Commitments** (Address → Comm) +**Section 6: Commitments** (Address → Comm) ``` count (Tag0) [Address (32 bytes) + secret_addr (32 bytes) + payload_addr (32 bytes)]* ``` +The full reader (`Env::get` / Lean `getEnv`) rejects trailing bytes +after Section 6. + +### Anonymous-only loading + +`Env::get_anon` (`crates/ixon/src/serialize.rs`) is a sibling of +`Env::get` that reads the header, §1 blobs, §2 consts, and §3 hints, +then STOPS — the metadata sections are laid out after the hints +precisely so anon readers never touch them. The returned `Env` has +empty `named`/`names`/`comms` and is suitable for anon-mode kernel +workflows that never consult metadata. Steady-state memory for a +Mathlib-scale env drops from ~3-4 GB (structured + metadata) to ~1 GB +(lazy bytes only). + +Exposed to Lean via `rs_de_env_anon` (`Ix.Ixon.rsDeEnvAnon`); +`Env::get_anon_mmap` is the zero-copy mmap sibling and +`Env::parse_lazy_index` the zero-copy index variant (reads through §5, +skipping only comms). + +### Bundles: pinning a single value + +A **bundle** is an `.ixe` whose `main` points at a distinguished +constant (e.g. an anonymous `Defn` wrapping a value, produced by +`Ix.Commit.compileDef`) and whose contents are closed up to +`assumptions`. Because a constant's address is a merkle root over its +entire dependency DAG (refs tables hold addresses of constants and +literal blobs, recursively), `main`'s 32 bytes alone pin the value; +the bundle is the data-availability artifact that ships the bytes +behind those addresses. + +- `Env::prune_to_closure(main, assumed)` builds a bundle: the 3-edge + closure of `main` (Expr refs; projection → `Muts` block; `Muts` + block → member/constructor projections), cut at `assumed`, carrying + constants (genuine bytes), blobs, per-constant hints, and display + metadata (named entries, name components + string blobs, `DataValue` + payload blobs, `meta_refs` extension edges, aux_gen originals). +- `Env::validate_closed()` is the receiver-side check: `main ∈ consts` + and every reachable address is carried (consts ∪ blobs) or assumed. +- Whole-environment files are the degenerate case (`main` absent, + `assumptions` empty). + +The CLI producer is `ix pack`: + +``` +lake exe ix pack [--out ] + [--assume ] [--assume-file ] [--verbose] +``` + +It resolves `` (displayed form) against the env's `named` table, +runs the prune, re-validates with `validate_closed`, and writes the +bundle (default `.ixe`). `--assume` entries — names or 64-hex +constant addresses — declare trust-boundary cut points; the ones +actually reached become the bundle's `assumptions` (thin bundle). + +The source env is memory-mapped and lazily loaded; display metadata is +carried by **re-streaming §5 per prune fixpoint round** +(`Env::prune_to_closure_streaming`) so resident metadata is +O(survivors), not O(env) — byte-identical output to the in-memory +`prune_to_closure`, which shares the same carry engine. `--anon` skips +metadata entirely (`Env::prune_to_closure_anon`): value closure + §3 +hints, empty §4/§5 — the minimal artifact a receiver needs to +typecheck/evaluate the pinned value, since `validate_closed` checks +only the value pin. (`ix shard extract` is the non-bundle sibling: a +general sub-env for the kernel-check pipeline, no `main` root.) + +### Diffing environments + +``` +lake exe ix diff [--anon | --meta] [--verbose] +``` + +`ix diff` (engine: `ixon::diff`) joins two envs on names: a name +"changed" ⇔ its constant address changed, with per-field +classification (`type`/`value`/`lvls`/…, `block.*` through projection +descent, `"encoding"` for pure representation churn). Because one +edited constant re-addresses its whole reverse-dependency cone, every +changed row also carries a **root vs rippled** verdict: changed pairs +are re-classified under the quotient of all changed rows' old→new +address mapping, and rows fully explained by dependency re-addressing +are `rippled` (hidden by default; `--verbose` lists). A 5-day mathlib +window classifies 143k changed names into ~4.5k roots. + +Memory: both files are mmap'd and lazily parsed in both modes +(constant windows stay zero-copy; `ConstantMeta` is never +bulk-materialized). `--meta` compares metadata by streaming both §5 +named sections in a lockstep merge-join — each side's entry is parsed +against its own §4 reverse index, compared, and dropped. Raw §5 +windows are *not* comparable across files (metadata name references +are file-relative §4 indices), so the sweep compares parsed, +Address-valued `ConstantMeta`. Exit codes follow GNU diff: 0 = no +difference in the selected mode, 1 = differences, 2 = error. + --- ## Proofs and Claims @@ -881,7 +993,7 @@ fits in single-byte tags (sizes 0..=7 per flag). | Size | Byte | Type | Payload | |------|------|------|---------| -| 0 | `0xE0` | Environment | bare 32-byte merkle root + 5 sections | +| 0 | `0xE0` | Environment | 32-byte merkle root + main/assumptions + 6 sections | | 1 | `0xE1` | Commitment | 2 addr: secret, payload | | 2 | `0xE2` | AssumptionTree | recursive merkle-tree body (see below) | | 3 | `0xE3` | Eval claim | 2 addr (input, output) + opt assumptions | @@ -1200,7 +1312,6 @@ Named { meta: ConstantMeta::Def { name: addr_of_name("double"), lvls: [], - hints: ReducibilityHints::Regular(1), all: [addr_of_name("double")], ctx: [], arena: ExprMeta { nodes: [