From 42f24dbfdd0ade849bb491c92229a86e6fbfc047 Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Sun, 5 Jul 2026 01:39:21 -0400 Subject: [PATCH 1/8] =?UTF-8?q?Ixon:=20.ixe=20bundle=20format=20=E2=80=94?= =?UTF-8?q?=20main/assumptions=20header,=20metadata-last=20sections,=20blo?= =?UTF-8?q?b=20integrity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evolves .ixe in place (Tag4(0xE,0) kept, pre-alpha) so a single env file can pin one Lean value as a self-contained bundle: - Header gains main : Option Address (bundle root; writers and all readers enforce main ∈ consts) and a strictly-ascending assumptions list (thin-bundle trust boundary; merkle_root_canonical over the leaves reproduces Claim.assumptions roots). - Sections reordered hot-first: blobs, consts, anon_hints, names, named, comms. get_anon/get_anon_mmap stop after §3 (no metadata parse-and-discard); parse_lazy_index stops after §5; the full readers reject trailing bytes. - anon_hints promoted from optional trailer to an always-written §3, derived from Named Def metadata when the map is empty; read-time hint harvesting removed (EnvHandle::from_bytes post-pass dropped). - Blob entries are hash-verified in every reader on both sides (a swapped blob silently changed a Nat/String literal before); Lean getEnv also gains the const hash check it was missing. - New Env::bfs_closure follows all three edge kinds (Constant.refs, Prj→block, Muts→member/ctor projections — a refs-only walk returns just the root for projections); prune_to_closure builds bundles (value closure + named-metadata pruning: name parent chains and string blobs, DataValue payload blobs, meta_refs edges, aux_gen originals, per-const hints); validate_closed is the receiver check. kernel::anon_work::closure_addrs now delegates to bfs_closure. - Lean topologicalSortNames emits the anonymous name at index 0, matching Rust: serEnv and rsSerEnv are now byte-identical (verified byte-exact on a 505 MB whole-stdlib env) and the putIdx getD-0 fallback now resolves to anon instead of colliding with the first real name. - FFI mirrors: RawEnv carries main/assumptions/anonHints (ctor arity 5→8), RawEnvLazy carries main/assumptions (3→5); the manual RawEnv builders in ffi compile.rs populate the new slots via set_raw_env_bundle_fields. - Generators/fixtures content-address blobs and exercise the bundle fields; tampering tests compute offsets programmatically instead of hardcoding the header layout; docs/Ixon.md spec updated. --- Ix/Ixon.lean | 204 ++++- Tests/FFI/Lifecycle.lean | 14 +- Tests/Gen/Ixon.lean | 63 +- Tests/Ix/Compile.lean | 8 +- Tests/Ix/Ixon.lean | 70 +- crates/ffi/src/compile.rs | 5 + crates/ffi/src/lean.rs | 8 +- crates/ffi/src/lean_ixon/env.rs | 155 +++- crates/ffi/src/lean_ixon/serialize.rs | 54 ++ crates/ixon/src/env.rs | 466 ++++++++++- crates/ixon/src/metadata.rs | 99 +++ crates/ixon/src/serialize.rs | 1081 +++++++++++++++++-------- crates/ixvm-codegen/src/env_handle.rs | 37 +- crates/kernel/src/anon_work.rs | 82 +- crates/kernel/src/ingress.rs | 18 +- docs/Ixon.md | 113 ++- 16 files changed, 1917 insertions(+), 560 deletions(-) diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index 0bf494acf..610c3ca57 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -1520,6 +1520,20 @@ structure Env where comms : Std.HashMap Address Comm := {} /-- Reverse index: constant Address → Ix.Name -/ addrToName : Std.HashMap Address Ix.Name := {} + /-- Distinguished root constant for bundle envs; `none` for whole + environments. A pointer, not a proof: readers check + `main ∈ consts`, and consumers holding an externally-expected + address must compare against it. -/ + main : Option Address := none + /-- Explicit trust boundary for thin bundles: addresses (constants + or blobs) the receiver is expected to already have. Serialized + as a strictly ascending leaf list; `Ix.Merkle.merkleRootCanonical` + over it reproduces the root a `Claim.assumptions` commits to. -/ + assumptions : Std.HashSet Address := {} + /-- Reducibility hints (§3, the canonical hint channel for anon/lazy + readers). When empty, `putEnv` derives the section from Named + `.defn` metadata at write time. Mirrors Rust `Env.anon_hints`. -/ + anonHints : Std.HashMap Address Lean.ReducibilityHints := {} deriving Inhabited namespace Env @@ -1627,13 +1641,21 @@ structure RawNameEntry where deriving Repr, Inhabited, BEq /-- Raw FFI environment structure using arrays instead of HashMaps. - This is the array-based equivalent of `Env` for FFI compatibility. -/ + This is the array-based equivalent of `Env` for FFI compatibility. + Field order matters: the Rust mirror (`LeanIxonRawEnv`) addresses + constructor slots positionally (0-7). -/ structure RawEnv where consts : Array RawConst named : Array RawNamed blobs : Array RawBlob comms : Array RawComm names : Array RawNameEntry := #[] + /-- Bundle root (`Env.main`). -/ + main : Option Address := none + /-- Bundle trust boundary (`Env.assumptions`), sorted ascending. -/ + assumptions : Array Address := #[] + /-- Explicit reducibility hints (`Env.anonHints`), sorted by address. -/ + anonHints : Array (Address × Lean.ReducibilityHints) := #[] deriving Repr, Inhabited, BEq namespace RawEnv @@ -1691,7 +1713,10 @@ def toEnv (raw : RawEnv) : Env := Id.run do env := { env with blobs := env.blobs.insert addr bytes } for ⟨addr, comm⟩ in raw.comms do env := env.storeComm addr comm - return env + return { env with + main := raw.main + assumptions := raw.assumptions.foldl (·.insert ·) {} + anonHints := raw.anonHints.foldl (fun m (a, h) => m.insert a h) {} } end RawEnv @@ -1700,7 +1725,9 @@ end RawEnv namespace Env /-- Convert Env with HashMaps to RawEnv with Arrays for FFI. - Includes the full names table for round-trip fidelity. -/ + Includes the full names table for round-trip fidelity. The + set-shaped bundle fields are sorted so the transfer is + deterministic (matches Rust `ixon_env_to_decoded`). -/ def toRawEnv (env : Env) : RawEnv := { consts := env.consts.toArray.map fun (addr, lc) => { addr, const := lc.get?.getD default } @@ -1708,6 +1735,11 @@ def toRawEnv (env : Env) : RawEnv := { blobs := env.blobs.toArray.map fun (addr, bytes) => { addr, bytes } comms := env.comms.toArray.map fun (addr, comm) => { addr, comm } names := env.names.toArray.map fun (addr, name) => { addr, name } + main := env.main + assumptions := env.assumptions.toList.toArray.qsort + fun a b => (compare a b).isLT + anonHints := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT } /-- Tag4 flag for Env (0xE), variant 0. -/ @@ -1773,11 +1805,15 @@ partial def topologicalSortNames (names : Std.HashMap Address Ix.Name) : Array ( let visited := visited.insert addr let result := result.push (addr, name) (visited, result) - -- Start with anonymous already visited (it's implicit) + -- Include the anonymous name first so it gets index 0 in the name + -- index (arena nodes frequently reference it as a binder name). + -- Matches Rust `topological_sort_names`, which emits it explicitly — + -- required for byte-identical writer output across the mirrors. let initVisited : Std.HashSet Address := ({} : Std.HashSet Address).insert anonAddr + let initResult : Array (Address × Ix.Name) := #[(anonAddr, Ix.Name.mkAnon)] -- Sort names by address before iterating to ensure deterministic DFS order let sortedEntries := names.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT - let (_, result) := sortedEntries.foldl (init := (initVisited, #[])) fun (visited, result) (_, name) => + let (_, result) := sortedEntries.foldl (init := (initVisited, initResult)) fun (visited, result) (_, name) => visit name visited result result @@ -1795,6 +1831,19 @@ def putEnv (env : Env) : PutM Unit := do let root := (Ix.Merkle.merkleRootCanonical constAddrs).getD Ix.Merkle.zeroAddress Serialize.put root + -- Bundle header fields: main (Option, 0/1-tagged) + assumptions + -- (strictly ascending address list). Matches Rust `Env::put`. + match env.main with + | none => putU8 0 + | some addr => do + putU8 1 + Serialize.put addr + let assumptions := env.assumptions.toList.toArray.qsort + fun a b => (compare a b).isLT + putTag0 ⟨assumptions.size.toUInt64⟩ + for addr in assumptions do + Serialize.put addr + -- Section 1: Blobs (Address -> bytes) let blobs := env.blobs.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨blobs.size.toUInt64⟩ @@ -1820,7 +1869,36 @@ def putEnv (env : Env) : PutM Unit := do putTag0 ⟨bytes.size.toUInt64⟩ putBytes bytes - -- Section 3: Names (Address -> Name component) + -- Section 3: anon_hints — the canonical hint channel for the + -- anon/lazy readers, placed before the metadata sections so they can + -- stop right after it. When the in-memory map is empty (the compile + -- path: hints live in Named metadata), the section is derived from + -- `named`. `named` is sorted here (the §5 canonical order, by name + -- hash) and reused for §5 below; the sorted iteration makes the + -- first-wins dedup by constant address deterministic. Matches Rust + -- `Env::put`. + let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT + let hintPairs : Array (Address × Lean.ReducibilityHints) := + if env.anonHints.isEmpty then Id.run do + let mut seen : Std.HashSet Address := {} + let mut pairs : Array (Address × Lean.ReducibilityHints) := #[] + for (_, namedEntry) in named do + match namedEntry.constMeta with + | .defn _ _ hints _ _ _ _ _ => + if !seen.contains namedEntry.addr then + seen := seen.insert namedEntry.addr + pairs := pairs.push (namedEntry.addr, hints) + | _ => pure () + return pairs + else + env.anonHints.toList.toArray + let hintPairs := hintPairs.qsort fun a b => (compare a.1 b.1).isLT + putTag0 ⟨hintPairs.size.toUInt64⟩ + for (addr, hints) in hintPairs do + Serialize.put addr + putReducibilityHints hints + + -- Section 4: Names (Address -> Name component) -- Topologically sorted so parents come before children, with ties broken by address let sortedNames := topologicalSortNames env.names -- Build name index from sorted positions (matching Rust) @@ -1831,8 +1909,7 @@ def putEnv (env : Env) : PutM Unit := do Serialize.put addr putNameComponent name - -- Section 4: Named (name Address -> Named with metadata) - let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT + -- Section 5: Named (name Address -> Named with metadata) putTag0 ⟨named.size.toUInt64⟩ for (name, namedEntry) in named do -- Use the name's stored hash, which matches how it was stored in env.names @@ -1847,7 +1924,7 @@ def putEnv (env : Env) : PutM Unit := do Serialize.put origAddr putConstantMetaIndexed origMeta nameIdx - -- Section 5: Comms (Address -> Comm) + -- Section 6: Comms (Address -> Comm) let comms := env.comms.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨comms.size.toUInt64⟩ for (addr, comm) in comms do @@ -1872,28 +1949,68 @@ def getEnv : GetM Env := do -- the recomputed root. let storedRoot : Address ← Serialize.get - let mut env : Env := {} - - -- Section 1: Blobs + -- Bundle header fields: main (Option) + strictly ascending + -- assumptions list. A pre-bundle-format `.ixe` has the §1 blob + -- count here, so a bad tag most likely means a stale file. + let mainTag ← getU8 + let main : Option Address ← match mainTag with + | 0 => pure none + | 1 => some <$> Serialize.get + | x => throw s!"Env.get: invalid main tag {x} in bundle header — \ + possibly a pre-bundle-format .ixe; recompile it" + let numAssumptions := (← getTag0).size + let mut assumptionArr : Array Address := #[] + for _ in [:numAssumptions.toNat] do + let addr : Address ← Serialize.get + if let some prev := assumptionArr.back? then + if !(compare prev addr).isLT then + throw "Env.get: assumptions not strictly ascending" + assumptionArr := assumptionArr.push addr + + let mut env : Env := { + main + assumptions := assumptionArr.foldl (·.insert ·) {} + } + + -- Section 1: Blobs (hash-verified per entry: a swapped blob would + -- otherwise silently change a Nat/String literal's value — the + -- consts merkle root covers only constant addresses) let numBlobs := (← getTag0).size for _ in [:numBlobs.toNat] do let addr ← Serialize.get let len := (← getTag0).size let bytes ← getBytes len.toNat + if Address.blake3 bytes != addr then + throw s!"Env.get: blob bytes hash mismatch for {reprStr (toString addr)}" env := { env with blobs := env.blobs.insert addr bytes } - -- Section 2: Consts (length-prefixed; see putEnv for rationale) + -- Section 2: Consts (length-prefixed; see putEnv for rationale). + -- Per-entry integrity: bytes must hash to the stored address. let numConsts := (← getTag0).size for _ in [:numConsts.toNat] do let addr ← Serialize.get let len := (← getTag0).size let bytes ← getBytes len.toNat + if Address.blake3 bytes != addr then + throw s!"Env.get: const bytes hash mismatch for {reprStr (toString addr)}" match deConstant bytes with | .ok constant => env := env.storeConst addr constant | .error e => throw s!"Env.get: bad constant bytes for addr {reprStr (toString addr)}: {e}" - -- Section 3: Names (build lookup table AND reverse index) + -- `main` must reference a constant actually present in the file. + if let some m := main then + if !env.consts.contains m then + throw s!"Env.get: main {reprStr (toString m)} not present in consts" + + -- Section 3: anon_hints + let numHints := (← getTag0).size + for _ in [:numHints.toNat] do + let addr : Address ← Serialize.get + let hints ← getReducibilityHints + env := { env with anonHints := env.anonHints.insert addr hints } + + -- Section 4: Names (build lookup table AND reverse index) let numNames := (← getTag0).size let mut namesLookup : Std.HashMap Address Ix.Name := {} let mut nameRev : NameReverseIndex := #[] @@ -1906,7 +2023,7 @@ def getEnv : GetM Env := do namesLookup := namesLookup.insert addr name env := { env with names := env.names.insert addr name } - -- Section 4: Named (name Address -> Named with metadata) + -- Section 5: Named (name Address -> Named with metadata) let numNamed := (← getTag0).size for _ in [:numNamed.toNat] do let nameAddr ← Serialize.get @@ -1930,7 +2047,7 @@ def getEnv : GetM Env := do | none => throw s!"getEnv: named entry references unknown name address {reprStr (toString nameAddr)}" - -- Section 5: Comms + -- Section 6: Comms let numComms := (← getTag0).size for _ in [:numComms.toNat] do let addr ← Serialize.get (α := Address) @@ -1946,6 +2063,13 @@ def getEnv : GetM Env := do if computedRoot != storedRoot then throw "Env.get: merkle root mismatch" + -- Comms is the final section; trailing bytes are truncation damage + -- or concatenated garbage. (Mirrors Rust `Env::get`; the early-stop + -- readers cannot make this check by design.) + let st ← get + if st.idx != st.bytes.size then + throw s!"Env.get: {st.bytes.size - st.idx} trailing bytes after final section" + pure env end Env @@ -1956,8 +2080,9 @@ def serEnv (env : Env) : ByteArray := runPut (Env.putEnv env) /-- Deserialize an Env from bytes (full metadata, pure Lean). -/ def deEnv (bytes : ByteArray) : Except String Env := runGet Env.getEnv bytes -/-- Compute section sizes for debugging. Returns (blobs, consts, names, named, comms). -/ -def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do +/-- Compute section sizes for debugging. + Returns (blobs, consts, anonHints, names, named, comms). -/ +def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat × Nat := Id.run do -- Blobs section let blobsBytes := runPut do let blobs := env.blobs.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT @@ -1975,6 +2100,29 @@ def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do Serialize.put addr putBytes lc.rawBytes + -- anon_hints section (mirrors putEnv's derive-from-named rule) + let hintsBytes := runPut do + let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT + let hintPairs : Array (Address × Lean.ReducibilityHints) := + if env.anonHints.isEmpty then Id.run do + let mut seen : Std.HashSet Address := {} + let mut pairs : Array (Address × Lean.ReducibilityHints) := #[] + for (_, namedEntry) in named do + match namedEntry.constMeta with + | .defn _ _ hints _ _ _ _ _ => + if !seen.contains namedEntry.addr then + seen := seen.insert namedEntry.addr + pairs := pairs.push (namedEntry.addr, hints) + | _ => pure () + return pairs + else + env.anonHints.toList.toArray + let hintPairs := hintPairs.qsort fun a b => (compare a.1 b.1).isLT + putTag0 ⟨hintPairs.size.toUInt64⟩ + for (addr, hints) in hintPairs do + Serialize.put addr + putReducibilityHints hints + -- Names section let namesBytes := runPut do let sortedNames := Env.topologicalSortNames env.names @@ -2009,7 +2157,8 @@ def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat := Id.run do Serialize.put addr putComm comm - (blobsBytes.size, constsBytes.size, namesBytes.size, namedBytes.size, commsBytes.size) + ( blobsBytes.size, constsBytes.size, hintsBytes.size, namesBytes.size, + namedBytes.size, commsBytes.size ) /-! ## Rust FFI Serialization -/ @@ -2056,11 +2205,16 @@ structure RawNamedLite where hintVal : UInt64 deriving Inhabited -/-- Metadata-light env returned by `rs_de_env_lazy`. -/ +/-- Metadata-light env returned by `rs_de_env_lazy`. Field order + matters: the Rust builder addresses constructor slots 0-4. -/ structure RawEnvLazy where consts : Array RawConstSlice named : Array RawNamedLite blobs : Array RawBlob + /-- Bundle root (`Env.main`) from the header. -/ + main : Option Address := none + /-- Bundle trust boundary (`Env.assumptions`), in header order. -/ + assumptions : Array Address := #[] deriving Inhabited /-- Decode an encoded reducibility hint into the `ConstantMeta` the check path @@ -2080,7 +2234,8 @@ def RawNamedLite.toConstMeta (n : RawNamedLite) : ConstantMeta := that share `buf`. `buf` MUST be the exact buffer passed to `rs_de_env_lazy` (offsets are relative to it). -/ def RawEnvLazy.toEnv (raw : RawEnvLazy) (buf : ByteArray) : Env := Id.run do - let mut env : Env := {} + let mut env : Env := { main := raw.main + assumptions := raw.assumptions.foldl (·.insert ·) {} } for ⟨addr, offset, len⟩ in raw.consts do env := { env with consts := env.consts.insert addr @@ -2104,9 +2259,10 @@ opaque rsDeEnvLazyFFI : @& ByteArray → Except String RawEnvLazy def deEnvAnon (bytes : ByteArray) : Except String Env := return (← rsDeEnvLazyFFI bytes).toEnv bytes -/-- Anonymous-only deserialization: keep blobs + consts, parse-and-drop - names/named/comms. Returns a `RawEnv` whose `named`/`names`/`comms` - arrays are empty. -/ +/-- Anonymous-only deserialization: keep blobs + consts + anon_hints + and stop — the metadata sections (names/named/comms) are laid out + after the hints and never touched. Returns a `RawEnv` whose + `named`/`names`/`comms` arrays are empty. -/ @[extern "rs_de_env_anon"] opaque rsDeEnvAnonFFI : @& ByteArray → Except String RawEnv diff --git a/Tests/FFI/Lifecycle.lean b/Tests/FFI/Lifecycle.lean index 790990383..d30554b78 100644 --- a/Tests/FFI/Lifecycle.lean +++ b/Tests/FFI/Lifecycle.lean @@ -137,10 +137,11 @@ def serdeTests : TestSeq := -- Empty RawEnv. Only data construction happens eagerly; FFI is deferred -- inside `mkSerdeRoundtripTest`. let empty : RawEnv := { consts := #[], named := #[], blobs := #[], comms := #[] } - -- RawEnv with data. The const's `addr` must be the canonical content - -- hash (`Address.blake3 (serConstant c)`) — the Rust loader verifies - -- `Address::hash(bytes) == addr` on load. Blobs and comms don't carry - -- a content-hash invariant, so `testAddr` is fine there. + -- RawEnv with data. Consts AND blobs must be content-addressed — the + -- Rust loader verifies `Address::hash(bytes) == addr` per entry for + -- both. `testAddr` is blake3 of `[1,2,3]`, which is exactly the blob's + -- bytes below, so the blob invariant holds; comms carry no + -- content-hash invariant, so `testAddr` is fine there too. let testAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) let testExpr : Expr := .sort 0 let testDef : Definition := { @@ -220,13 +221,12 @@ private def genSerdeRawEnv : Gen RawEnv := do let idx ← Gen.choose Nat 0 (pool.size - 1) let (addr, name) := pool[idx]! named := named.push { name, addr, constMeta := .empty } - -- Blobs: pool addresses + -- Blobs: content-addressed (the loaders verify blake3(bytes) == addr) let numBlobs ← Gen.choose Nat 0 3 let mut blobs : Array RawBlob := #[] for _ in [:numBlobs] do - let addr ← pickAddr let bytes ← Tests.Gen.Ixon.genByteArray - blobs := blobs.push { addr, bytes } + blobs := blobs.push { addr := Address.blake3 bytes, bytes } -- Comms: pool addresses let numComms ← Gen.choose Nat 0 2 let mut comms : Array RawComm := #[] diff --git a/Tests/Gen/Ixon.lean b/Tests/Gen/Ixon.lean index 4f7678449..b001c0372 100644 --- a/Tests/Gen/Ixon.lean +++ b/Tests/Gen/Ixon.lean @@ -457,9 +457,12 @@ def genRawConst : Gen RawConst := do def genRawNamed : Gen RawNamed := RawNamed.mk <$> genIxName 3 <*> genAddress <*> pure .empty -/-- Generate a RawBlob -/ -def genRawBlob : Gen RawBlob := - RawBlob.mk <$> genAddress <*> genByteArray +/-- Generate a RawBlob whose `addr` is the canonical content hash of + `bytes`. Readers verify `Address.blake3 bytes == addr` per entry + (blob-tampering defense), so arbitrary addresses are rejected. -/ +def genRawBlob : Gen RawBlob := do + let bytes ← genByteArray + pure { addr := Address.blake3 bytes, bytes } /-- Generate a RawComm -/ def genRawComm : Gen RawComm := @@ -469,13 +472,29 @@ def genRawComm : Gen RawComm := def genRawNameEntry : Gen RawNameEntry := RawNameEntry.mk <$> genAddress <*> genIxName 3 -/-- Generate a RawEnv with small arrays to avoid memory issues -/ -def genRawEnv : Gen RawEnv := - RawEnv.mk <$> genSmallArray genRawConst - <*> genSmallArray genRawNamed - <*> genSmallArray genRawBlob - <*> genSmallArray genRawComm - <*> genSmallArray genRawNameEntry +/-- Generate a RawEnv with small arrays to avoid memory issues. `main` + (when present) references a generated const — writers/readers check + `main ∈ consts`. Assumptions and hint keys are opaque addresses. -/ +def genRawEnv : Gen RawEnv := do + let consts ← genSmallArray genRawConst + let named ← genSmallArray genRawNamed + let blobs ← genSmallArray genRawBlob + let comms ← genSmallArray genRawComm + let names ← genSmallArray genRawNameEntry + let main ← + if consts.size > 0 then + frequency [ + (1, pure none), + (1, do + let i ← Gen.choose Nat 0 (consts.size - 1) + pure (consts[i % consts.size]?.map (·.addr))), + ] + else pure none + let assumptionList ← genList genAddress + let assumptions := assumptionList.toArray.qsort fun a b => (compare a b).isLT + let hintList ← genList (Prod.mk <$> genAddress <*> genReducibilityHints) + let anonHints := hintList.toArray.qsort fun a b => (compare a.1 b.1).isLT + pure { consts, named, blobs, comms, names, main, assumptions, anonHints } instance : Shrinkable RawConst where shrink rc := (fun c => { rc with const := c }) <$> Shrinkable.shrink rc.const @@ -486,18 +505,36 @@ instance : Shrinkable RawNamed where | _ => [{ rn with constMeta := .empty }] instance : Shrinkable RawBlob where - shrink rb := if rb.bytes.size > 0 then [{ rb with bytes := ByteArray.empty }] else [] + shrink rb := + -- Keep the content-address invariant: shrunk bytes need the + -- matching hash or the loaders reject the shrunk case. + if rb.bytes.size > 0 then + [{ addr := Address.blake3 ByteArray.empty, bytes := ByteArray.empty }] + else [] instance : Shrinkable RawComm where shrink _ := [] instance : Shrinkable RawEnv where shrink env := - (if env.consts.size > 0 then [{ env with consts := env.consts.pop }] else []) ++ + -- Popping a const may orphan `main` (writers reject main ∉ consts), + -- so clear it when its target is dropped. + (if env.consts.size > 0 then + let popped := env.consts.pop + let main := match env.main with + | some m => if popped.any (·.addr == m) then env.main else none + | none => none + [{ env with consts := popped, main }] + else []) ++ (if env.named.size > 0 then [{ env with named := env.named.pop }] else []) ++ (if env.blobs.size > 0 then [{ env with blobs := env.blobs.pop }] else []) ++ (if env.comms.size > 0 then [{ env with comms := env.comms.pop }] else []) ++ - (if env.names.size > 0 then [{ env with names := env.names.pop }] else []) + (if env.names.size > 0 then [{ env with names := env.names.pop }] else []) ++ + (if env.assumptions.size > 0 then [{ env with assumptions := env.assumptions.pop }] else []) ++ + (if env.anonHints.size > 0 then [{ env with anonHints := env.anonHints.pop }] else []) ++ + (match env.main with + | some _ => [{ env with main := none }] + | none => []) instance : SampleableExt RawConst := SampleableExt.mkSelfContained genRawConst instance : SampleableExt RawNamed := SampleableExt.mkSelfContained genRawNamed diff --git a/Tests/Ix/Compile.lean b/Tests/Ix/Compile.lean index f3d34e9cf..7efdf0003 100644 --- a/Tests/Ix/Compile.lean +++ b/Tests/Ix/Compile.lean @@ -337,8 +337,8 @@ def testCrossImpl : TestSeq := IO.println s!"[Step 4] Lean blob stats: total={fmtBytes leanTotalBlobData}, max={fmtBytes leanMaxBlob}, avg={fmtBytes leanAvgBlob}, big(>1kB)={leanBig}, huge(>100kB)={leanHuge}" IO.println s!"[Step 4] Lean top 10 blob sizes: {leanTopSizes.map fmtBytes}" - let (leanBlobs, leanConsts, leanNames, leanNamed, leanComms) := Ixon.envSectionSizes leanIxonEnv - IO.println s!"[Step 4] Lean sections: blobs={fmtBytes leanBlobs}, consts={fmtBytes leanConsts}, names={fmtBytes leanNames}, named={fmtBytes leanNamed}, comms={fmtBytes leanComms}" + let (leanBlobs, leanConsts, leanHints, leanNames, leanNamed, leanComms) := Ixon.envSectionSizes leanIxonEnv + IO.println s!"[Step 4] Lean sections: blobs={fmtBytes leanBlobs}, consts={fmtBytes leanConsts}, hints={fmtBytes leanHints}, names={fmtBytes leanNames}, named={fmtBytes leanNamed}, comms={fmtBytes leanComms}" let leanEnvBytes := serializeEnv leanIxonEnv IO.println s!"[Step 4] Lean env done: {fmtBytes leanEnvBytes.size}" @@ -358,8 +358,8 @@ def testCrossImpl : TestSeq := IO.println s!"[Step 4] Rust blob stats: total={fmtBytes rustTotalBlobData}, max={fmtBytes rustMaxBlob}, avg={fmtBytes rustAvgBlob}, big(>1kB)={rustBig}, huge(>100kB)={rustHuge}" IO.println s!"[Step 4] Rust top 10 blob sizes: {rustTopSizes.map fmtBytes}" - let (rustBlobs, rustConsts, rustNames, rustNamed, rustComms) := Ixon.envSectionSizes phases.compileEnv - IO.println s!"[Step 4] Rust sections: blobs={fmtBytes rustBlobs}, consts={fmtBytes rustConsts}, names={fmtBytes rustNames}, named={fmtBytes rustNamed}, comms={fmtBytes rustComms}" + let (rustBlobs, rustConsts, rustHints, rustNames, rustNamed, rustComms) := Ixon.envSectionSizes phases.compileEnv + IO.println s!"[Step 4] Rust sections: blobs={fmtBytes rustBlobs}, consts={fmtBytes rustConsts}, hints={fmtBytes rustHints}, names={fmtBytes rustNames}, named={fmtBytes rustNamed}, comms={fmtBytes rustComms}" let rustEnvBytes := serializeEnv phases.compileEnv IO.println s!"[Step 4] Rust env done: {fmtBytes rustEnvBytes.size}" let serTime := (← IO.monoMsNow) - serStart diff --git a/Tests/Ix/Ixon.lean b/Tests/Ix/Ixon.lean index 72d22ec48..db7394bdd 100644 --- a/Tests/Ix/Ixon.lean +++ b/Tests/Ix/Ixon.lean @@ -153,9 +153,11 @@ def envSerdeUnit (env : Env) : Bool := def envUnitTests : TestSeq := -- Test 1: Empty env let emptyEnv : Env := {} - -- Test 2: Env with only a blob - let blobAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) - let envWithBlob : Env := { blobs := ({} : Std.HashMap _ _).insert blobAddr (ByteArray.mk #[4, 5, 6]) } + -- Test 2: Env with only a blob. Blob addresses must be the content + -- hash of the bytes — readers verify per entry. + let blobBytes := ByteArray.mk #[4, 5, 6] + let blobAddr := Address.blake3 blobBytes + let envWithBlob : Env := { blobs := ({} : Std.HashMap _ _).insert blobAddr blobBytes } -- Test 3: Env with a simple name (no named entry) let testName := Ix.Name.mkStr Ix.Name.mkAnon "test" let testNameAddr := testName.getHash @@ -179,15 +181,34 @@ def envUnitTests : TestSeq := let payloadAddr := Address.blake3 (ByteArray.mk #[13, 14, 15]) let commAddr := Address.blake3 (ByteArray.mk #[16, 17, 18]) let envWithBlobAndComm : Env := { - blobs := ({} : Std.HashMap _ _).insert blobAddr (ByteArray.mk #[4, 5, 6]), + blobs := ({} : Std.HashMap _ _).insert blobAddr blobBytes, comms := ({} : Std.HashMap _ _).insert commAddr (Comm.mk secretAddr payloadAddr) } + -- Test 7: Bundle env — main + assumptions + anonHints exercise the + -- bundle header and §3. + let bundleDef : Definition := { + kind := .defn, safety := .safe, lvls := 0, typ := .sort 0, value := .sort 0 + } + let bundleConst : Constant := { + info := .defn bundleDef, sharing := #[], refs := #[], univs := #[] + } + let bundleConstAddr := Address.blake3 (serConstant bundleConst) + let bundleEnv : Env := Id.run do + let mut env : Env := {} + env := env.storeConst bundleConstAddr bundleConst + return { env with + main := some bundleConstAddr + assumptions := ({} : Std.HashSet Address) + |>.insert (Address.blake3 (ByteArray.mk #[42])) + |>.insert (Address.blake3 (ByteArray.mk #[43])) + anonHints := ({} : Std.HashMap _ _).insert bundleConstAddr (.regular 5) } test "Empty env roundtrip" (envSerdeUnit emptyEnv) ++ test "Env with blob roundtrip" (envSerdeUnit envWithBlob) ++ test "Env with name roundtrip" (envSerdeUnit envWithName) ++ test "Env with named (empty meta) roundtrip" (envSerdeUnit envWithNamed) ++ test "Env with nested name roundtrip" (envSerdeUnit envWithNestedName) ++ - test "Env with blob+comm roundtrip" (envSerdeUnit envWithBlobAndComm) + test "Env with blob+comm roundtrip" (envSerdeUnit envWithBlobAndComm) ++ + test "Bundle env (main/assumptions/hints) roundtrip" (envSerdeUnit bundleEnv) /-! ## Cross-implementation serialization comparison tests -/ @@ -204,15 +225,22 @@ def envSerializationMatches (raw : RawEnv) : Bool := let env := raw.toEnv rsEqEnvSerialization raw (serEnv env) +/-- Strict byte equality between the pure-Lean writer and the Rust + writer over the same RawEnv (a stronger check than + `rsEqEnvSerialization`'s content comparison). -/ +def envBytesMatchRust (raw : RawEnv) : Bool := + serEnv raw.toEnv == rsSerEnvFFI raw + /-- Unit tests for Lean==Rust serialization comparison -/ def envSerializationUnitTests : TestSeq := -- Test 1: Empty env let emptyRaw : RawEnv := { consts := #[], named := #[], blobs := #[], comms := #[] } - -- Test 2: Env with one blob - let blobAddr := Address.blake3 (ByteArray.mk #[1, 2, 3]) + -- Test 2: Env with one blob (content-addressed: readers verify). + let blobBytes := ByteArray.mk #[4, 5, 6] + let blobAddr := Address.blake3 blobBytes let blobRaw : RawEnv := { consts := #[], named := #[], - blobs := #[{ addr := blobAddr, bytes := ByteArray.mk #[4, 5, 6] }], + blobs := #[{ addr := blobAddr, bytes := blobBytes }], comms := #[] } -- Test 3: Env with one comm @@ -227,13 +255,35 @@ def envSerializationUnitTests : TestSeq := -- Test 4: Env with blob + comm let blobCommRaw : RawEnv := { consts := #[], named := #[], - blobs := #[{ addr := blobAddr, bytes := ByteArray.mk #[4, 5, 6] }], + blobs := #[{ addr := blobAddr, bytes := blobBytes }], comms := #[{ addr := commAddr, comm := Comm.mk secretAddr payloadAddr }] } + -- Test 5: Bundle env — the directed cross-language vector for the + -- bundle header (main/assumptions) and §3 anon_hints. + let bundleDef : Definition := { + kind := .defn, safety := .safe, lvls := 0, typ := .sort 0, value := .sort 0 + } + let bundleConst : Constant := { + info := .defn bundleDef, sharing := #[], refs := #[], univs := #[] + } + let bundleConstAddr := Address.blake3 (serConstant bundleConst) + let bundleRaw : RawEnv := { + consts := #[{ addr := bundleConstAddr, const := bundleConst }], + named := #[], blobs := #[], comms := #[], + main := some bundleConstAddr, + assumptions := #[ + Address.blake3 (ByteArray.mk #[42]), + Address.blake3 (ByteArray.mk #[43])], + anonHints := #[(bundleConstAddr, .regular 5)] + } test "Empty env Lean==Rust" (envSerializationMatches emptyRaw) ++ test "Blob env Lean==Rust" (envSerializationMatches blobRaw) ++ test "Comm env Lean==Rust" (envSerializationMatches commRaw) ++ - test "Blob+Comm env Lean==Rust" (envSerializationMatches blobCommRaw) + test "Blob+Comm env Lean==Rust" (envSerializationMatches blobCommRaw) ++ + test "Bundle env Lean==Rust" (envSerializationMatches bundleRaw) ++ + test "Empty env bytes Lean==Rust" (envBytesMatchRust emptyRaw) ++ + test "Blob env bytes Lean==Rust" (envBytesMatchRust blobRaw) ++ + test "Bundle env bytes Lean==Rust" (envBytesMatchRust bundleRaw) /-! ## Canonical env merkle root: Lean vs. Rust agreement -/ diff --git a/crates/ffi/src/compile.rs b/crates/ffi/src/compile.rs index 23fb3f74b..f3a9234e5 100644 --- a/crates/ffi/src/compile.rs +++ b/crates/ffi/src/compile.rs @@ -443,6 +443,10 @@ pub extern "C" fn rs_compile_phases( raw_ixon_env.set_obj(2, blobs_arr); raw_ixon_env.set_obj(3, comms_arr); raw_ixon_env.set_obj(4, names_arr); + crate::lean_ixon::env::set_raw_env_bundle_fields( + &raw_ixon_env, + &compile_stt.env, + ); let result = LeanIxCompilePhases::alloc(0); result.set_obj(0, raw_env); @@ -539,6 +543,7 @@ pub extern "C" fn rs_compile_env_to_ixon( result.set_obj(2, blobs_arr); result.set_obj(3, comms_arr); result.set_obj(4, names_arr); + crate::lean_ixon::env::set_raw_env_bundle_fields(&result, &compile_stt.env); LeanIOResult::ok(result) } } diff --git a/crates/ffi/src/lean.rs b/crates/ffi/src/lean.rs index 0fc8d29e9..cb920c9f9 100644 --- a/crates/ffi/src/lean.rs +++ b/crates/ffi/src/lean.rs @@ -52,13 +52,15 @@ lean_ffi::lean_inductive! { LeanIxonRawBlob [ { num_obj: 2 } ]; LeanIxonRawComm [ { num_obj: 2 } ]; LeanIxonRawNameEntry [ { num_obj: 2 } ]; - LeanIxonRawEnv [ { num_obj: 5 } ]; + // consts, named, blobs, comms, names, main, assumptions, anonHints + LeanIxonRawEnv [ { num_obj: 8 } ]; // Lazy/anon deserialization (`rs_de_env_lazy`): zero-copy const windows - // (addr + offset + len), name->addr + hint, and copied blobs. + // (addr + offset + len), name->addr + hint, copied blobs, and the + // bundle header fields (main, assumptions). LeanIxonRawConstSlice [ { num_obj: 1, num_64: 2 } ]; LeanIxonRawNamedLite [ { num_obj: 2, num_64: 2 } ]; - LeanIxonRawEnvLazy [ { num_obj: 3 } ]; + LeanIxonRawEnvLazy [ { num_obj: 5 } ]; // --- Ixon multi-variant inductives --- diff --git a/crates/ffi/src/lean_ixon/env.rs b/crates/ffi/src/lean_ixon/env.rs index edac79ad2..e2a5dcd88 100644 --- a/crates/ffi/src/lean_ixon/env.rs +++ b/crates/ffi/src/lean_ixon/env.rs @@ -4,10 +4,10 @@ //! RawConst, RawNamed, RawBlob, RawComm. use crate::lean::{ - LeanIxName, LeanIxonComm, LeanIxonConstant, LeanIxonConstantMeta, - LeanIxonRawBlob, LeanIxonRawComm, LeanIxonRawConst, LeanIxonRawConstSlice, - LeanIxonRawEnv, LeanIxonRawEnvLazy, LeanIxonRawNameEntry, LeanIxonRawNamed, - LeanIxonRawNamedLite, + LeanIxName, LeanIxReducibilityHints, LeanIxonComm, LeanIxonConstant, + LeanIxonConstantMeta, LeanIxonRawBlob, LeanIxonRawComm, LeanIxonRawConst, + LeanIxonRawConstSlice, LeanIxonRawEnv, LeanIxonRawEnvLazy, + LeanIxonRawNameEntry, LeanIxonRawNamed, LeanIxonRawNamedLite, }; use ix_common::address::Address; use ix_common::env::{Name, ReducibilityHints}; @@ -17,7 +17,8 @@ use ixon::env::{Env as IxonEnv, LazyIndex, Named as IxonNamed}; use ixon::merkle::merkle_root_canonical; use ixon::metadata::ConstantMeta; use lean_ffi::object::{ - LeanArray, LeanBorrowed, LeanByteArray, LeanExcept, LeanOwned, LeanRef, + LeanArray, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, LeanOwned, + LeanProd, LeanRef, }; use crate::builder::LeanBuildCache; @@ -226,7 +227,7 @@ impl LeanIxonRawNameEntry { } // ============================================================================= -// RawEnv (consts, named, blobs, comms, names) +// RawEnv (consts, named, blobs, comms, names, main, assumptions, anonHints) // ============================================================================= /// Decoded Ixon.RawEnv @@ -236,6 +237,12 @@ pub struct DecodedRawEnv { pub blobs: Vec, pub comms: Vec, pub names: Vec, + /// Bundle root (`Env::main`). + pub main: Option
, + /// Bundle trust boundary (`Env::assumptions`), sorted ascending. + pub assumptions: Vec
, + /// Explicit reducibility hints (`Env::anon_hints`), sorted by address. + pub anon_hints: Vec<(Address, ReducibilityHints)>, } impl LeanIxonRawEnv { @@ -274,6 +281,27 @@ impl LeanIxonRawEnv { .set(i, LeanIxonRawNameEntry::build(&mut cache, &rn.addr, &rn.name)); } + // Build bundle fields: main (Option Address), assumptions + // (Array Address), anonHints (Array (Address × ReducibilityHints)). + let main_obj: LeanOwned = match &env.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let assumptions_arr = LeanArray::alloc(env.assumptions.len()); + for (i, addr) in env.assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + let hints_arr = LeanArray::alloc(env.anon_hints.len()); + for (i, (addr, hint)) in env.anon_hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + // Build RawEnv structure let ctor = LeanIxonRawEnv::alloc(0); ctor.set_obj(0, consts_arr); @@ -281,10 +309,51 @@ impl LeanIxonRawEnv { ctor.set_obj(2, blobs_arr); ctor.set_obj(3, comms_arr); ctor.set_obj(4, names_arr); + ctor.set_obj(5, main_obj); + ctor.set_obj(6, assumptions_arr); + ctor.set_obj(7, hints_arr); ctor } } +/// Populate the bundle-field constructor slots (5-7: main, +/// assumptions, anonHints) of a `RawEnv` being built field-by-field — +/// the compile FFI assembles its RawEnv manually instead of going +/// through [`LeanIxonRawEnv::build`], and leaving these slots unset is +/// uninitialized memory (segfault on GC). Set-shaped fields are sorted +/// for a deterministic transfer. +pub fn set_raw_env_bundle_fields( + ctor: &LeanIxonRawEnv, + env: &IxonEnv, +) { + let main_obj: LeanOwned = match &env.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let mut assumptions: Vec
= env.assumptions.iter().cloned().collect(); + assumptions.sort_unstable(); + let assumptions_arr = LeanArray::alloc(assumptions.len()); + for (i, addr) in assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + let mut hints: Vec<(Address, ReducibilityHints)> = + env.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect(); + hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let hints_arr = LeanArray::alloc(hints.len()); + for (i, (addr, hint)) in hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + ctor.set_obj(5, main_obj); + ctor.set_obj(6, assumptions_arr); + ctor.set_obj(7, hints_arr); +} + impl LeanIxonRawEnv { /// Decode Ixon.RawEnv from Lean pointer. pub fn decode(&self) -> DecodedRawEnv { @@ -294,6 +363,23 @@ impl LeanIxonRawEnv { let blobs_arr = ctor.get(2).as_array(); let comms_arr = ctor.get(3).as_array(); let names_arr = ctor.get(4).as_array(); + let main_obj = ctor.get(5); + let assumptions_arr = ctor.get(6).as_array(); + let hints_arr = ctor.get(7).as_array(); + + // `Option Address`: scalar-optimized none, or a tag-0/1 ctor. + let main: Option
= if main_obj.is_scalar() { + None + } else { + let opt = main_obj.as_ctor(); + match opt.tag() { + 0 => None, + 1 => Some( + LeanIxAddress::from_borrowed(opt.get(0).as_byte_array()).decode(), + ), + tag => panic!("Invalid Option tag for RawEnv.main: {tag}"), + } + }; DecodedRawEnv { consts: consts_arr @@ -304,6 +390,17 @@ impl LeanIxonRawEnv { comms: comms_arr.map(|x| LeanIxonRawComm::new(x.to_owned_ref()).decode()), names: names_arr .map(|x| LeanIxonRawNameEntry::new(x.to_owned_ref()).decode()), + main, + assumptions: assumptions_arr + .map(|x| LeanIxAddress::from_borrowed(x.as_byte_array()).decode()), + anon_hints: hints_arr.map(|x| { + let pair = x.as_ctor(); + let addr = + LeanIxAddress::from_borrowed(pair.get(0).as_byte_array()).decode(); + let hint = + LeanIxReducibilityHints::new(pair.get(1).to_owned_ref()).decode(); + (addr, hint) + }), } } } @@ -314,7 +411,7 @@ impl LeanIxonRawEnv { /// Reconstruct a Rust IxonEnv from a DecodedRawEnv. pub fn decoded_to_ixon_env(decoded: &DecodedRawEnv) -> IxonEnv { - let env = IxonEnv::new(); + let mut env = IxonEnv::new(); for rc in &decoded.consts { env.store_const(rc.addr.clone(), rc.constant.clone()); } @@ -331,6 +428,11 @@ pub fn decoded_to_ixon_env(decoded: &DecodedRawEnv) -> IxonEnv { for rc in &decoded.comms { env.store_comm(rc.addr.clone(), rc.comm.clone()); } + env.main = decoded.main.clone(); + env.assumptions.extend(decoded.assumptions.iter().cloned()); + env + .anon_hints + .extend(decoded.anon_hints.iter().map(|(a, h)| (a.clone(), *h))); env } @@ -380,7 +482,22 @@ pub fn ixon_env_to_decoded(env: &IxonEnv) -> Result { name: e.value().clone(), }) .collect(); - Ok(DecodedRawEnv { consts, named, blobs, comms, names }) + // Sort the set-shaped fields so the FFI transfer is deterministic. + let mut assumptions: Vec
= env.assumptions.iter().cloned().collect(); + assumptions.sort_unstable(); + let mut anon_hints: Vec<(Address, ReducibilityHints)> = + env.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect(); + anon_hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + Ok(DecodedRawEnv { + consts, + named, + blobs, + comms, + names, + main: env.main.clone(), + assumptions, + anon_hints, + }) } // ============================================================================= @@ -451,11 +568,12 @@ pub extern "C" fn rs_de_env( /// FFI: Anonymous-only deserialization (`Env::get_anon`). /// -/// Reads the header + blobs + consts sections; parses and discards -/// the metadata sections (names / named / comms). The returned -/// `RawEnv` has empty `named`, `names`, `comms` arrays. Useful for -/// anon-mode kernel callers that want to avoid the steady-state -/// memory cost of metadata that they will never consult. +/// Reads the header + §1 blobs + §2 consts + §3 anon_hints and stops; +/// the metadata sections (names / named / comms) are laid out after +/// the hints and never touched. The returned `RawEnv` has empty +/// `named`, `names`, `comms` arrays. Useful for anon-mode kernel +/// callers that want to avoid the steady-state memory cost of +/// metadata they will never consult. #[unsafe(no_mangle)] pub extern "C" fn rs_de_env_anon( obj: LeanByteArray>, @@ -552,10 +670,21 @@ fn build_raw_env_lazy(index: &LazyIndex) -> LeanIxonRawEnvLazy { blobs_arr.set(i, LeanIxonRawBlob::build_from_parts(addr, bytes)); } + let main_obj: LeanOwned = match &index.main { + None => LeanOption::none().into(), + Some(addr) => LeanOption::some(LeanIxAddress::build(addr)).into(), + }; + let assumptions_arr = LeanArray::alloc(index.assumptions.len()); + for (i, addr) in index.assumptions.iter().enumerate() { + assumptions_arr.set(i, LeanIxAddress::build(addr)); + } + let ctor = LeanIxonRawEnvLazy::alloc(0); ctor.set_obj(0, consts_arr); ctor.set_obj(1, named_arr); ctor.set_obj(2, blobs_arr); + ctor.set_obj(3, main_obj); + ctor.set_obj(4, assumptions_arr); ctor } diff --git a/crates/ffi/src/lean_ixon/serialize.rs b/crates/ffi/src/lean_ixon/serialize.rs index 68061c03a..e681802ac 100644 --- a/crates/ffi/src/lean_ixon/serialize.rs +++ b/crates/ffi/src/lean_ixon/serialize.rs @@ -234,6 +234,60 @@ pub extern "C" fn rs_eq_env_serialization( } } + // Bundle header fields. + if rust_env.main != decoded.main { + if debug { + eprintln!( + "[rs_eq_env_serialization] main mismatch: rust={:?}, decoded={:?}", + rust_env.main.as_ref().map(Address::hex), + decoded.main.as_ref().map(Address::hex), + ); + } + return false; + } + let decoded_assumptions: rustc_hash::FxHashSet
= + decoded.assumptions.iter().cloned().collect(); + if rust_env.assumptions != decoded_assumptions { + if debug { + eprintln!( + "[rs_eq_env_serialization] assumptions mismatch: rust={}, decoded={}", + rust_env.assumptions.len(), + decoded_assumptions.len(), + ); + } + return false; + } + + // Hints: the writers derive §3 from Named `Def` metadata when the + // explicit map is empty, so the parsed env's hints must equal the + // same derivation over the decoded RawEnv. + let expected_hints: rustc_hash::FxHashMap< + Address, + ix_common::env::ReducibilityHints, + > = if decoded.anon_hints.is_empty() { + let mut derived = rustc_hash::FxHashMap::default(); + for rn in &decoded.named { + if let ixon::metadata::ConstantMetaInfo::Def { hints, .. } = + &rn.const_meta.info + { + derived.entry(rn.addr.clone()).or_insert(*hints); + } + } + derived + } else { + decoded.anon_hints.iter().cloned().collect() + }; + if rust_env.anon_hints != expected_hints { + if debug { + eprintln!( + "[rs_eq_env_serialization] anon_hints mismatch: rust={}, expected={}", + rust_env.anon_hints.len(), + expected_hints.len(), + ); + } + return false; + } + true } diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index 3a5f46f58..c17fb9010 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -8,7 +8,10 @@ use ix_common::address::Address; use ix_common::env::{Name, ReducibilityHints}; use super::comm::Comm; -use super::constant::Constant; +use super::constant::{ + Constant, ConstantInfo, MutConst, ctor_proj_address, defn_proj_address, + indc_proj_address, recr_proj_address, +}; use super::lazy::LazyConstant; use super::map::IxonMap; use super::metadata::{ConstantMeta, ConstantMetaInfo}; @@ -196,6 +199,10 @@ pub struct LazyIndex { pub consts: Vec, pub named: Vec, pub blobs: Vec<(Address, Vec)>, + /// Bundle root (`Env::main`) as read from the header. + pub main: Option
, + /// Bundle trust boundary (`Env::assumptions`), in header (sorted) order. + pub assumptions: Vec
, } /// The Ixon environment. @@ -235,6 +242,18 @@ pub struct Env { /// supplying them in anon mode does not relax the kernel's /// metadata-free correctness model. pub anon_hints: FxHashMap, + /// Distinguished root constant for bundle envs; `None` for whole + /// environments. A pointer, not a proof: nothing in the file + /// authenticates it, so readers check `main ∈ consts` and consumers + /// holding an externally-expected address (from a Claim, a request) + /// must compare against it. + pub main: Option
, + /// Explicit trust boundary for thin bundles: addresses (constants + /// or blobs) the receiver is expected to already have, so the + /// closure of `main` need not be carried in full. Serialized as a + /// strictly ascending leaf list; `merkle_root_canonical` over it + /// reproduces the root a `Claim::assumptions` field commits to. + pub assumptions: FxHashSet
, } impl Env { @@ -246,6 +265,8 @@ impl Env { names: IxonMap::new(), comms: IxonMap::new(), anon_hints: FxHashMap::default(), + main: None, + assumptions: FxHashSet::default(), } } @@ -477,6 +498,281 @@ impl Env { v.sort_unstable(); v } + + /// Collect `c`'s outgoing closure edges: Expr-level `refs`, a + /// projection's structural `block` pointer, and — for a `Muts` + /// block at `addr` — every member/constructor projection address + /// (derived, not stored anywhere; `ingress_anon_block` computes + /// these and requires them present in the env). + fn closure_edges(addr: &Address, c: &Constant, edges: &mut Vec
) { + edges.extend(c.refs.iter().cloned()); + match &c.info { + ConstantInfo::IPrj(p) => edges.push(p.block.clone()), + ConstantInfo::CPrj(p) => edges.push(p.block.clone()), + ConstantInfo::RPrj(p) => edges.push(p.block.clone()), + ConstantInfo::DPrj(p) => edges.push(p.block.clone()), + ConstantInfo::Muts(members) => { + for (i, m) in members.iter().enumerate() { + let i = i as u64; + edges.push(match m { + MutConst::Defn(_) => defn_proj_address(i, addr), + MutConst::Indc(_) => indc_proj_address(i, addr), + MutConst::Recr(_) => recr_proj_address(i, addr), + }); + if let MutConst::Indc(ind) = m { + for cidx in 0..ind.ctors.len() as u64 { + edges.push(ctor_proj_address(i, cidx, addr)); + } + } + } + }, + _ => {}, + } + } + + /// BFS-collect the full dependency closure of `roots`, following + /// all three structural edge kinds (see [`Self::closure_edges`]): + /// + /// 1. `Constant.refs` — Expr-level references (constants and blobs); + /// 2. projection → its `Muts` block (`DPrj`/`IPrj`/`RPrj`/`CPrj` + /// carry `block` in the info payload and have EMPTY refs tables, + /// so a refs-only walk returns just the projection itself); + /// 3. `Muts` block → every member/constructor projection address. + /// + /// Compare [`Self::bfs_refs`], which follows only edge 1 and stays + /// the basis of claim assumption roots. The kernel's + /// `anon_work::closure_addrs` delegates here. + /// + /// Referenced addresses absent from `consts` (blobs and external + /// assumptions) appear in the returned set as leaves. + pub fn bfs_closure(&self, roots: &[Address]) -> FxHashSet
{ + let mut closure: FxHashSet
= FxHashSet::default(); + let mut queue: VecDeque
= VecDeque::new(); + for r in roots { + if closure.insert(r.clone()) { + queue.push_back(r.clone()); + } + } + while let Some(addr) = queue.pop_front() { + // Materialize just long enough to read edges; drop the map + // guard before recursing (see `bfs_refs`). + let constant = self.consts.get(&addr).and_then(|r| r.value().get().ok()); + let Some(c) = constant else { continue }; + let mut edges: Vec
= Vec::new(); + Self::closure_edges(&addr, &c, &mut edges); + for e in edges { + if closure.insert(e.clone()) { + queue.push_back(e); + } + } + } + closure + } + + /// Bundle completeness check (receiver side): `main` must be a + /// stored constant, and every address reachable from it via + /// [`Self::bfs_closure`] must be either carried by this env + /// (consts ∪ blobs) or declared in `assumptions`. + /// + /// This validates the VALUE pin. Display-metadata completeness + /// (names, `DataValue` blobs) is the writer's business — + /// [`Self::prune_to_closure`] carries it; a receiver that only + /// typechecks/evaluates never needs it. + pub fn validate_closed(&self) -> Result<(), String> { + let Some(main) = self.main.clone() else { + return Err("validate_closed: env has no main".to_string()); + }; + if self.consts.get(&main).is_none() { + return Err(format!( + "validate_closed: main {} not present in consts", + main.hex() + )); + } + for addr in self.bfs_closure(std::slice::from_ref(&main)) { + if self.consts.contains_key(&addr) + || self.blobs.contains_key(&addr) + || self.assumptions.contains(&addr) + { + continue; + } + return Err(format!( + "validate_closed: {} reachable from main but neither carried nor \ + assumed", + addr.hex() + )); + } + Ok(()) + } + + /// Build a self-contained bundle env: the 3-edge closure of `main`, + /// cut at `assumed`. + /// + /// - Reached constants are carried with their GENUINE bytes + /// (`store_const_lazy`), so the receiver's per-entry hash check + /// and consts merkle root hold; reached blobs are copied. + /// - Cut-points actually reached go to `out.assumptions` (minimal: + /// a declared-but-unreached assumption is not carried). + /// - `anon_hints` are copied per carried constant (else a bundle + /// regresses kernel-check time to the `Regular(0)` fallback). + /// - Display metadata is carried for every carried constant: its + /// `named` entries (all of them — alpha-equivalent names may + /// share one address), the name components they reference (with + /// full parent chains and string-component blobs), `DataValue` + /// payload blobs, `meta_refs` extension DAG edges, and aux_gen + /// `original` constants. Metadata can introduce new DAG edges, so + /// the walk runs to fixpoint (usually two rounds). + /// + /// Errors if a reached address is in neither `consts`, `blobs`, nor + /// `assumed` — the source env cannot produce a closed bundle for + /// `main` under that cut. + /// + /// Host-only — see `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure( + &self, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + if assumed.contains(main) { + return Err("prune_to_closure: main cannot be assumed".to_string()); + } + let mut out = Env::new(); + out.main = Some(main.clone()); + + let mut visited: FxHashSet
= FxHashSet::default(); + let mut pending: VecDeque
= VecDeque::new(); + visited.insert(main.clone()); + pending.push_back(main.clone()); + let mut named_done: FxHashSet = FxHashSet::default(); + + loop { + // ── Value pass: 3-edge BFS over pending roots, cut at `assumed`. + while let Some(addr) = pending.pop_front() { + if assumed.contains(&addr) { + out.assumptions.insert(addr); + continue; + } + if let Some(bytes) = self.get_const_bytes(&addr) { + out.store_const_lazy(addr.clone(), bytes); + if let Some(h) = self.anon_hints.get(&addr) { + out.anon_hints.insert(addr.clone(), *h); + } + let c = self.get_const(&addr).ok_or_else(|| { + format!("prune_to_closure: constant {} unparseable", addr.hex()) + })?; + let mut edges: Vec
= Vec::new(); + Self::closure_edges(&addr, &c, &mut edges); + for e in edges { + if visited.insert(e.clone()) { + pending.push_back(e); + } + } + } else if let Some(blob) = self.get_blob(&addr) { + out.blobs.insert(addr, blob); + } else { + return Err(format!( + "prune_to_closure: {} reachable from main but not in \ + consts/blobs and not assumed", + addr.hex() + )); + } + } + + // ── Named pass: carry display metadata for every carried + // constant. Metadata references content the value walk cannot + // see; new DAG edges feed the next value pass. + for entry in self.named.iter() { + let named = entry.value(); + if !out.consts.contains_key(&named.addr) + || named_done.contains(entry.key()) + { + continue; + } + named_done.insert(entry.key().clone()); + out.named.insert(entry.key().clone(), named.clone()); + self.carry_name(&mut out, entry.key()); + + let mut name_addrs: Vec
= Vec::new(); + let mut blob_addrs: Vec
= Vec::new(); + let mut dag_addrs: Vec
= Vec::new(); + named.meta().collect_deps( + &mut name_addrs, + &mut blob_addrs, + &mut dag_addrs, + ); + if let Some((orig_addr, orig_meta)) = named.original() { + dag_addrs.push(orig_addr); + orig_meta.collect_deps( + &mut name_addrs, + &mut blob_addrs, + &mut dag_addrs, + ); + } + for na in name_addrs { + let Some(name) = self.get_name(&na) else { + return Err(format!( + "prune_to_closure: metadata references name {} absent from \ + names", + na.hex() + )); + }; + self.carry_name(&mut out, &name); + } + for ba in blob_addrs { + let Some(blob) = self.get_blob(&ba) else { + return Err(format!( + "prune_to_closure: metadata references blob {} absent from \ + blobs", + ba.hex() + )); + }; + out.blobs.insert(ba, blob); + } + for da in dag_addrs { + if visited.insert(da.clone()) { + pending.push_back(da); + } + } + } + + // The named pass ran against the final consts of this round; if + // it produced no new DAG work, the walk is complete. + if pending.is_empty() { + break; + } + } + Ok(out) + } + + /// Copy `name` and its full parent chain into `out.names`, storing + /// each string component's bytes as a blob (the compiler's + /// convention — mirrors `addNameComponentsWithBlobs` on the Lean + /// side). Stops early once a component is already present: its + /// parents were carried with it. + #[cfg(not(target_arch = "riscv64"))] + fn carry_name(&self, out: &mut Env, name: &Name) { + use ix_common::env::NameData; + let mut cur = name.clone(); + loop { + let addr = Address::from_blake3_hash(*cur.get_hash()); + if out.names.get(&addr).is_some() { + return; + } + out.names.insert(addr, cur.clone()); + let next = match cur.as_data() { + NameData::Anonymous(_) => None, + NameData::Str(parent, s, _) => { + out.store_blob(s.as_bytes().to_vec()); + Some(parent.clone()) + }, + NameData::Num(parent, _, _) => Some(parent.clone()), + }; + match next { + Some(p) => cur = p, + None => return, + } + } + } } impl Clone for Env { @@ -516,6 +812,8 @@ impl Clone for Env { names, comms, anon_hints: self.anon_hints.clone(), + main: self.main.clone(), + assumptions: self.assumptions.clone(), } } } @@ -912,4 +1210,170 @@ mod tests { ); } } + + // --------------------------------------------------------------------------- + // bfs_closure / prune_to_closure / validate_closed + // --------------------------------------------------------------------------- + + fn defn_member(discriminator: u64) -> crate::constant::Definition { + use crate::constant::{DefKind, Definition}; + use ix_common::env::DefinitionSafety; + Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls: discriminator, + typ: Arc::new(Expr::Sort(0)), + value: Arc::new(Expr::Var(0)), + } + } + + #[test] + fn bfs_closure_follows_projection_and_block_edges() { + use crate::constant::{MutConst, defn_proj_address, defn_proj_constant}; + let env = Env::new(); + let block_addr = store_canonical( + &env, + Constant::new(ConstantInfo::Muts(vec![ + MutConst::Defn(defn_member(0)), + MutConst::Defn(defn_member(1)), + ])), + ); + // Member projections live at derived addresses. + let p0_addr = + store_canonical(&env, defn_proj_constant(0, block_addr.clone())); + assert_eq!(p0_addr, defn_proj_address(0, &block_addr)); + let p1_addr = + store_canonical(&env, defn_proj_constant(1, block_addr.clone())); + + // From a projection root: the block and its sibling projections + // are reachable via the structural edges. + let closure = env.bfs_closure(std::slice::from_ref(&p0_addr)); + assert!(closure.contains(&p0_addr)); + assert!(closure.contains(&block_addr), "Prj → block edge"); + assert!(closure.contains(&p1_addr), "block → sibling projection edge"); + + // A refs-only walk sees none of this (projections have empty refs). + let refs_only = env.bfs_refs(&p0_addr); + assert_eq!(refs_only.len(), 1, "refs-only walk stops at the projection"); + } + + #[test] + fn prune_to_closure_carries_value_closure_and_blobs() { + let env = Env::new(); + let blob_addr = env.store_blob(b"forty two".to_vec()); + let c = store_canonical(&env, const_with_refs(vec![])); + let a = store_canonical( + &env, + const_with_refs(vec![blob_addr.clone(), c.clone()]), + ); + let d = store_canonical(&env, const_with_refs_discriminator(vec![], 7)); + + let bundle = env.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + assert_eq!(bundle.main, Some(a.clone())); + assert!(bundle.consts.contains_key(&a)); + assert!(bundle.consts.contains_key(&c)); + assert!(!bundle.consts.contains_key(&d), "unreachable const not carried"); + assert_eq!(bundle.get_blob(&blob_addr), Some(b"forty two".to_vec())); + assert!(bundle.assumptions.is_empty()); + bundle.validate_closed().unwrap(); + + // Bundle survives a serialize → deserialize roundtrip closed + // (genuine bytes: per-entry hashes and the merkle root hold). + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let loaded = Env::get(&mut buf.as_slice()).unwrap(); + loaded.validate_closed().unwrap(); + assert_eq!(loaded.main, Some(a)); + } + + #[test] + fn prune_to_closure_cuts_at_assumed() { + let env = Env::new(); + let c = store_canonical(&env, const_with_refs(vec![])); + let b = store_canonical(&env, const_with_refs(vec![c.clone()])); + let a = store_canonical(&env, const_with_refs(vec![b.clone()])); + let assumed: FxHashSet
= [b.clone()].into_iter().collect(); + let bundle = env.prune_to_closure(&a, &assumed).unwrap(); + assert!(bundle.consts.contains_key(&a)); + assert!(!bundle.consts.contains_key(&b), "cut point not carried"); + assert!(!bundle.consts.contains_key(&c), "beyond the cut not carried"); + assert_eq!( + bundle.assumptions, assumed, + "minimal assumptions = reached cuts" + ); + bundle.validate_closed().unwrap(); + } + + #[test] + fn prune_to_closure_missing_dep_errors_unless_assumed() { + let env = Env::new(); + let ghost = Address::hash(b"ghost"); + let a = store_canonical(&env, const_with_refs(vec![ghost.clone()])); + let err = env.prune_to_closure(&a, &FxHashSet::default()).unwrap_err(); + assert!(err.contains("not in consts/blobs"), "got: {err}"); + let assumed: FxHashSet
= [ghost].into_iter().collect(); + let bundle = env.prune_to_closure(&a, &assumed).unwrap(); + bundle.validate_closed().unwrap(); + } + + #[test] + fn validate_closed_rejects_missing_blob_unless_assumed() { + let mut env = Env::new(); + let blob_addr = Address::hash(b"blob-bytes"); + let a = store_canonical(&env, const_with_refs(vec![blob_addr.clone()])); + env.main = Some(a); + let err = env.validate_closed().unwrap_err(); + assert!(err.contains("reachable from main"), "got: {err}"); + env.assumptions.insert(blob_addr); + env.validate_closed().unwrap(); + } + + #[test] + fn prune_to_closure_carries_named_metadata() { + use crate::metadata::{ConstantMetaInfo, ExprMeta}; + let env = Env::new(); + let a = store_canonical(&env, const_with_refs(vec![])); + // A blob referenced ONLY from metadata (meta_refs extension table) + // — invisible to the value walk. + let meta_blob = env.store_blob(b"callsite payload".to_vec()); + let name = n("Bundled"); + let name_addr = Address::from_blake3_hash(*name.get_hash()); + env.store_name(name_addr.clone(), name.clone()); + let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { + name: name_addr.clone(), + lvls: vec![], + hints: ReducibilityHints::Regular(3), + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }); + meta.meta_refs.push(meta_blob.clone()); + env.register_name(name.clone(), Named::new(a.clone(), meta)); + + let bundle = env.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + assert!(bundle.named.get(&name).is_some(), "named entry carried"); + assert_eq!( + bundle.get_name(&name_addr), + Some(name.clone()), + "name component carried" + ); + assert_eq!( + bundle.get_blob(&meta_blob), + Some(b"callsite payload".to_vec()), + "meta_refs blob carried" + ); + // String-component blob carried too (compiler convention). + assert_eq!( + bundle.get_blob(&Address::hash(b"Bundled")), + Some(b"Bundled".to_vec()) + ); + // Hints derive from the carried Def meta on serialization; the + // anon reader sees them without touching metadata sections. + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!(anon.anon_hints.get(&a), Some(&ReducibilityHints::Regular(3))); + } } diff --git a/crates/ixon/src/metadata.rs b/crates/ixon/src/metadata.rs index c126fafb0..d6557f372 100644 --- a/crates/ixon/src/metadata.rs +++ b/crates/ixon/src/metadata.rs @@ -252,6 +252,105 @@ impl ConstantMeta { || !self.meta_univs.is_empty() } + /// Enumerate every external address this metadata references, + /// partitioned by the table that resolves it: + /// + /// - `names`: name-component addresses (resolved against + /// `Env.names`) — the variant's `name`/`lvls`/`all`/`ctx`/... + /// fields, arena binder/ref/proj/call-site names, and KVMap keys + /// plus `DataValue::OfName` payloads; + /// - `blobs`: raw-byte payload addresses (`DataValue` + /// strings/nats/ints/syntax, resolved against `Env.blobs`); + /// - `dag`: `meta_refs` extension-table addresses — constants or + /// blobs referenced by collapsed call-site argument expressions, + /// i.e. genuine value-DAG edges the primary `Constant.refs` walk + /// cannot see. + /// + /// Used by `Env::prune_to_closure` to carry a bundle's display + /// metadata completely. Duplicates are not filtered; callers dedup. + pub fn collect_deps( + &self, + names: &mut Vec
, + blobs: &mut Vec
, + dag: &mut Vec
, + ) { + use ConstantMetaInfo as I; + let mut arena: Option<&ExprMeta> = None; + match &self.info { + I::Empty => {}, + I::Def { name, lvls, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Axio { name, lvls, arena: a, .. } + | I::Quot { name, lvls, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + arena = Some(a); + }, + I::Indc { name, lvls, ctors, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(ctors.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Ctor { name, lvls, induct, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.push(induct.clone()); + arena = Some(a); + }, + I::Rec { name, lvls, rules, all, ctx, arena: a, .. } => { + names.push(name.clone()); + names.extend(lvls.iter().cloned()); + names.extend(rules.iter().cloned()); + names.extend(all.iter().cloned()); + names.extend(ctx.iter().cloned()); + arena = Some(a); + }, + I::Muts { all, .. } => { + for class in all { + names.extend(class.iter().cloned()); + } + }, + } + if let Some(a) = arena { + for node in &a.nodes { + match node { + ExprMetaData::Leaf | ExprMetaData::App { .. } => {}, + ExprMetaData::Binder { name, .. } + | ExprMetaData::LetBinder { name, .. } + | ExprMetaData::Ref { name } + | ExprMetaData::CallSite { name, .. } => names.push(name.clone()), + ExprMetaData::Prj { struct_name, .. } => { + names.push(struct_name.clone()); + }, + ExprMetaData::Mdata { mdata, .. } => { + for kv in mdata { + for (key, value) in kv { + names.push(key.clone()); + match value { + DataValue::OfName(a) => names.push(a.clone()), + DataValue::OfString(a) + | DataValue::OfNat(a) + | DataValue::OfInt(a) + | DataValue::OfSyntax(a) => blobs.push(a.clone()), + DataValue::OfBool(_) => {}, + } + } + } + }, + } + } + } + dag.extend(self.meta_refs.iter().cloned()); + } + /// Delegate indexed serialization to the inner enum, then serialize /// extension tables. pub fn put_with( diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index 08ee0856e..0d5da7cf6 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -76,25 +76,143 @@ fn put_address(a: &Address, buf: &mut Vec) { put_bytes(a.as_bytes(), buf); } -/// Read the optional trailing `anon_hints` section written by `Env::put`. -/// No-op when no bytes remain after the comms section — backward-compatible -/// with `.ixe` that have no hints or predate the section. Inserts into -/// `env.anon_hints` (overwriting any value harvested from a Named section, -/// with the identical value, so the order of harvest vs. section is moot). -fn read_anon_hints_section( +/// Write an `Option
` as `[0x00]` (None) or `[0x01][addr:32]` +/// (Some). Mirrors the Claim-side encoding in `proof.rs`. +fn put_opt_addr(opt: &Option
, buf: &mut Vec) { + match opt { + None => buf.push(0x00), + Some(addr) => { + buf.push(0x01); + buf.extend_from_slice(addr.as_bytes()); + }, + } +} + +fn get_opt_addr(buf: &mut &[u8]) -> Result, String> { + match get_u8(buf)? { + 0x00 => Ok(None), + 0x01 => Ok(Some(get_address(buf)?)), + b => Err(format!("get_opt_addr: invalid tag 0x{:02X}", b)), + } +} + +/// Read the `.ixe` header shared by every Env reader: `Tag4(0xE, 0)`, +/// the 32-byte consts merkle root, the bundle `main` pointer, and the +/// strictly ascending assumptions list. Centralized so the four readers +/// (`get`, `get_anon`, `get_anon_mmap`, `parse_lazy_index`) cannot +/// drift. `ctx` labels errors with the calling reader. +fn read_env_header( buf: &mut &[u8], - env: &mut Env, -) -> Result<(), String> { - if buf.is_empty() { - return Ok(()); + ctx: &str, +) -> Result<(Address, Option
, Vec
), String> { + let tag = Tag4::get(buf)?; + if tag.flag != Env::FLAG { + return Err(format!( + "{ctx}: expected flag 0x{:X}, got 0x{:X}", + Env::FLAG, + tag.flag + )); + } + if tag.size != 0 { + return Err(format!("{ctx}: expected Env variant 0, got {}", tag.size)); + } + let stored_root = get_address(buf)?; + // A pre-bundle-format `.ixe` has the §1 blob count here, so this + // byte is that count's Tag0 head — flag the likely cause when it + // isn't a valid opt tag. (.ixe files are regenerated artifacts.) + let main = get_opt_addr(buf).map_err(|e| { + format!( + "{ctx}: {e} in bundle header — possibly a pre-bundle-format .ixe; \ + recompile it" + ) + })?; + let n = get_u64(buf)? as usize; + // Each assumption is 32 bytes; a count the remaining buffer cannot + // hold is corruption (or a stale format) — reject before allocating. + if n > buf.len() / 32 { + return Err(format!( + "{ctx}: assumption count {n} exceeds remaining buffer — corrupt or \ + pre-bundle-format .ixe" + )); + } + let mut assumptions: Vec
= Vec::with_capacity(n); + for i in 0..n { + let addr = get_address(buf)?; + if let Some(prev) = assumptions.last() + && *prev >= addr + { + return Err(format!( + "{ctx}: assumptions not strictly ascending at idx {i} ({} then {})", + prev.hex(), + addr.hex() + )); + } + assumptions.push(addr); + } + Ok((stored_root, main, assumptions)) +} + +/// Read the §1 blob section, verifying `Address::hash(bytes) == addr` +/// per entry. Without the check a swapped blob would silently change a +/// Nat/String literal's value under an otherwise-valid file (the consts +/// merkle root covers only const addresses). +fn read_blob_section( + buf: &mut &[u8], + ctx: &str, +) -> Result)>, String> { + let num_blobs = get_u64(buf)? as usize; + // Each blob entry needs at least addr (32) + a length byte. + if num_blobs > buf.len() / 33 { + return Err(format!( + "{ctx}: blob count {num_blobs} exceeds remaining buffer" + )); + } + let mut blobs = Vec::with_capacity(num_blobs); + for i in 0..num_blobs { + let addr = get_address(buf)?; + let len = get_u64(buf)? as usize; + if buf.len() < len { + return Err(format!( + "{ctx}: need {} bytes for blob, have {}", + len, + buf.len() + )); + } + let (bytes, rest) = buf.split_at(len); + *buf = rest; + let computed = Address::hash(bytes); + if computed != addr { + return Err(format!( + "{ctx}: blob at idx {i} bytes hash to {} but stored under {}", + computed.hex(), + addr.hex() + )); + } + blobs.push((addr, bytes.to_vec())); } - let n = get_u64(buf)?; + Ok(blobs) +} + +/// Read the §3 `anon_hints` section (unconditional — every writer +/// emits it, deriving the entries from Named metadata when the +/// in-memory map is empty; see `Env::put`). +fn read_hints_section( + buf: &mut &[u8], +) -> Result, String> { + let n = get_u64(buf)? as usize; + // Each hint entry needs at least addr (32) + one byte of hint. + if n > buf.len() / 33 { + return Err(format!( + "read_hints_section: hint count {n} exceeds remaining buffer" + )); + } + let mut hints = Vec::with_capacity(n); for _ in 0..n { let addr = get_address(buf)?; - let hints = ReducibilityHints::get_ser(buf)?; - env.anon_hints.insert(addr, hints); + let hint = ReducibilityHints::get_ser(buf)?; + hints.push((addr, hint)); } - Ok(()) + Ok(hints) } fn get_address(buf: &mut &[u8]) -> Result { @@ -1153,12 +1271,32 @@ impl Env { let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, buf); + // ───────────────────────────────────────────────────────────────────── + // Bundle header fields: distinguished root + assumed-present list + // (see `Env::main` / `Env::assumptions`). Writer-side sanity: a + // `main` outside `consts` would produce a file every reader + // rejects — fail here with the clearer message. + // ───────────────────────────────────────────────────────────────────── + if let Some(m) = &self.main + && self.consts.get(m).is_none() + { + return Err(format!("Env::put: main {} not present in consts", m.hex())); + } + put_opt_addr(&self.main, buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, buf); + for addr in &assumption_addrs { + put_address(addr, buf); + } + // ───────────────────────────────────────────────────────────────────── // Section 1: Blobs (Address -> bytes) // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 1/5 blobs: {} entries", self.blobs.len(),); + eprintln!("[Env::put] section 1/6 blobs: {} entries", self.blobs.len(),); } let mut blob_addrs: Vec
= self.blobs.iter().map(|e| e.key().clone()).collect(); @@ -1177,7 +1315,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 1/5 blobs done in {:.1}s ({} bytes so far)", + "[Env::put] section 1/6 blobs done in {:.1}s ({} bytes so far)", sec_start.elapsed().as_secs_f64(), buf.len(), ); @@ -1191,11 +1329,11 @@ impl Env { // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 2/5 consts: {} entries", self.consts.len(),); + eprintln!("[Env::put] section 2/6 consts: {} entries", self.consts.len(),); } if !quiet { eprintln!( - "[Env::put] section 2/5 consts: collected+sorted in {:.1}s, \ + "[Env::put] section 2/6 consts: collected+sorted in {:.1}s, \ streaming put...", sec_start.elapsed().as_secs_f64(), ); @@ -1216,7 +1354,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 2/5 consts done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 2/6 consts done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1225,7 +1363,67 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 3: Names (Address -> Name component, topologically sorted) + // Section 3: anon_hints (Address -> ReducibilityHints) + // + // The canonical hint channel for the anon/lazy readers, placed + // before the metadata sections so `get_anon`/`get_anon_mmap` can + // stop right after it (they never touch names/named/comms). When + // the in-memory map is empty (the compile path: hints live in + // Named metadata), the section is derived from `named` — the same + // harvest `get_anon` historically did at read time, moved to + // write time. Hints are performance-only advice and intentionally + // NOT covered by the consts merkle root. + // + // `named_keys` is collected and sorted here (by name hash — the + // Section 5 canonical order) and reused for Section 5 below; the + // sorted iteration makes the first-wins dedup by constant address + // deterministic when alpha-equivalent names share one constant. + // ───────────────────────────────────────────────────────────────────── + let sec_start = std::time::Instant::now(); + let mut named_keys: Vec = + self.named.iter().map(|e| e.key().clone()).collect(); + #[cfg(not(target_arch = "riscv64"))] + named_keys.par_sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); + #[cfg(target_arch = "riscv64")] + named_keys.sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + if self.anon_hints.is_empty() { + let mut derived: FxHashMap = + FxHashMap::default(); + for name in &named_keys { + if let Some(entry) = self.named.get(name) + && let super::metadata::ConstantMetaInfo::Def { hints, .. } = + &entry.value().meta().info + { + derived.entry(entry.value().addr.clone()).or_insert(*hints); + } + } + derived.into_iter().collect() + } else { + self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() + }; + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, buf); + for (addr, hints) in &hint_pairs { + put_address(addr, buf); + hints.put_ser(buf); + } + if !quiet { + eprintln!( + "[Env::put] section 3/6 anon_hints done: {} entries in {:.1}s \ + ({} bytes so far)", + hint_pairs.len(), + sec_start.elapsed().as_secs_f64(), + buf.len(), + ); + } + + // ───────────────────────────────────────────────────────────────────── + // Section 4: Names (Address -> Name component, topologically sorted) // ───────────────────────────────────────────────────────────────────── // Topological sort ensures parents come before children so the name // index assigned during serialization is valid for all references that @@ -1234,14 +1432,14 @@ impl Env { let sec_start = std::time::Instant::now(); if !quiet { eprintln!( - "[Env::put] section 3/5 names: topo-sorting {} entries", + "[Env::put] section 4/6 names: topo-sorting {} entries", self.names.len(), ); } let sorted_names = topological_sort_names(&self.names); if !quiet { eprintln!( - "[Env::put] section 3/5 names: topo-sorted in {:.1}s, serializing...", + "[Env::put] section 4/6 names: topo-sorted in {:.1}s, serializing...", sec_start.elapsed().as_secs_f64(), ); } @@ -1255,7 +1453,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 3/5 names done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 4/6 names done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1264,37 +1462,18 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 4: Named (Name -> Named metadata with indexed addresses) + // Section 5: Named (Name -> Named metadata with indexed addresses) // ───────────────────────────────────────────────────────────────────── // Named values are the *largest* per-entry (each carries a ConstantMeta // with metadata arenas), so the streaming pattern's win is greatest // here: on Mathlib, avoiding the clone-into-Vec saves ~30 GB peak RAM. // - // Key clone cost: a `Name` is `Arc`, so each clone is a - // single atomic refcount increment (<1s for 733k). + // `named_keys` was collected and sorted (by cached name hash bytes) + // up in Section 3, where the hint derivation needs the same + // canonical order. let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 4/5 named: {} entries", self.named.len(),); - } - let mut named_keys: Vec = - self.named.iter().map(|e| e.key().clone()).collect(); - // Sort by the cached name hash bytes (same key used by every existing - // Section 4 ordering guarantee). `par_sort_unstable_by` uses rayon to - // parallelize the compare across all cores. - #[cfg(not(target_arch = "riscv64"))] - named_keys.par_sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); - #[cfg(target_arch = "riscv64")] - named_keys.sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); - if !quiet { - eprintln!( - "[Env::put] section 4/5 named: collected+sorted in {:.1}s, \ - streaming put...", - sec_start.elapsed().as_secs_f64(), - ); + eprintln!("[Env::put] section 5/6 named: {} entries", self.named.len(),); } let put_start = std::time::Instant::now(); put_u64(named_keys.len() as u64, buf); @@ -1306,7 +1485,7 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 4/5 named done: put in {:.1}s, total {:.1}s \ + "[Env::put] section 5/6 named done: put in {:.1}s, total {:.1}s \ ({} bytes so far)", put_start.elapsed().as_secs_f64(), sec_start.elapsed().as_secs_f64(), @@ -1315,11 +1494,11 @@ impl Env { } // ───────────────────────────────────────────────────────────────────── - // Section 5: Comms (Address -> Comm) — typically empty on compile path + // Section 6: Comms (Address -> Comm) — typically empty on compile path // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); if !quiet { - eprintln!("[Env::put] section 5/5 comms: {} entries", self.comms.len(),); + eprintln!("[Env::put] section 6/6 comms: {} entries", self.comms.len(),); } let mut comm_addrs: Vec
= self.comms.iter().map(|e| e.key().clone()).collect(); @@ -1336,33 +1515,12 @@ impl Env { } if !quiet { eprintln!( - "[Env::put] section 5/5 comms done in {:.1}s ({} bytes so far)", + "[Env::put] section 6/6 comms done in {:.1}s ({} bytes so far)", sec_start.elapsed().as_secs_f64(), buf.len(), ); } - // Optional trailing section: anon_hints (Address -> ReducibilityHints). - // `get_anon` normally HARVESTS hints from the Named section; a closure - // sub-env (built for shard injection) DROPS that section, which would - // force the kernel to `Regular(0)` and add a def-eq overhead vs whole-env - // proving. Carrying the hints here lets the guest reproduce vanilla kernel - // behavior exactly. Written only when populated, so compiler-produced - // `.ixe` (empty map) are byte-identical to before; loaders read it only if - // bytes remain after comms. Hints are performance-only (the `Regular(0)` - // fallback is always correct), so this section is intentionally NOT covered - // by the consts merkle root. - if !self.anon_hints.is_empty() { - let mut hint_addrs: Vec
= - self.anon_hints.keys().cloned().collect(); - hint_addrs.sort_unstable(); - put_u64(hint_addrs.len() as u64, buf); - for addr in &hint_addrs { - put_address(addr, buf); - self.anon_hints[addr].put_ser(buf); - } - } - if !quiet { eprintln!( "[Env::put] ALL DONE: {} bytes in {:.1}s", @@ -1411,6 +1569,25 @@ impl Env { let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, &mut buf); + // Bundle header fields: distinguished root + assumed-present list + // (mirrors `Env::put`, including the writer-side main sanity check). + if let Some(m) = &self.main + && self.consts.get(m).is_none() + { + return Err(format!( + "Env::put_file: main {} not present in consts", + m.hex() + )); + } + put_opt_addr(&self.main, &mut buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, &mut buf); + for addr in &assumption_addrs { + put_address(addr, &mut buf); + } + // Section 1: Blobs let mut blob_addrs: Vec
= self.blobs.iter().map(|e| e.key().clone()).collect(); @@ -1440,7 +1617,40 @@ impl Env { written += bytes.len() as u64; } } - // Section 3: Names (topologically sorted; builds the name index the + // Section 3: anon_hints — the canonical hint channel. Always + // emitted; when the in-memory map is empty it is derived from Named + // `Def` metadata in hash-sorted name order, mirroring `Env::put` + // exactly (same iteration order → same `or_insert` winners). + let mut named_keys: Vec = + self.named.iter().map(|e| e.key().clone()).collect(); + named_keys.par_sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + if self.anon_hints.is_empty() { + let mut derived: FxHashMap = + FxHashMap::default(); + for name in &named_keys { + if let Some(entry) = self.named.get(name) + && let super::metadata::ConstantMetaInfo::Def { hints, .. } = + &entry.value().meta().info + { + derived.entry(entry.value().addr.clone()).or_insert(*hints); + } + } + derived.into_iter().collect() + } else { + self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() + }; + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, &mut buf); + for (addr, hints) in &hint_pairs { + put_address(addr, &mut buf); + hints.put_ser(&mut buf); + emit!(); + } + + // Section 4: Names (topologically sorted; builds the name index the // Named section encodes through). let sorted_names = topological_sort_names(&self.names); let mut name_index: NameIndex = NameIndex::new(); @@ -1452,16 +1662,11 @@ impl Env { emit!(); } - // Section 4: Named — the largest per-entry section, and the only + // Section 5: Named — the largest per-entry section, and the only // one whose per-entry encode is CPU-heavy (demoted entries decode // and re-encode through the name index). Chunks encode in parallel // into per-entry buffers and drain to the writer in order; the // chunk size bounds staged memory to a few MiB. - let mut named_keys: Vec = - self.named.iter().map(|e| e.key().clone()).collect(); - named_keys.par_sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); put_u64(named_keys.len() as u64, &mut buf); emit!(); for chunk in named_keys.chunks(4096) { @@ -1481,7 +1686,7 @@ impl Env { written += b.len() as u64; } } - // Section 5: Comms + // Section 6: Comms let mut comm_addrs: Vec
= self.comms.iter().map(|e| e.key().clone()).collect(); comm_addrs.par_sort_unstable(); @@ -1494,64 +1699,31 @@ impl Env { } } - // Optional trailing anon_hints section — same condition as `put`. - if !self.anon_hints.is_empty() { - let mut hint_addrs: Vec
= - self.anon_hints.keys().cloned().collect(); - hint_addrs.sort_unstable(); - put_u64(hint_addrs.len() as u64, &mut buf); - for addr in &hint_addrs { - put_address(addr, &mut buf); - self.anon_hints[addr].put_ser(&mut buf); - } - } - emit!(); w.flush().map_err(|e| format!("Env::put_file: flush: {e}"))?; Ok(written) } /// Deserialize an Env from bytes. + /// + /// The whole buffer must be consumed — trailing bytes after the + /// final section are rejected (truncation/concatenation guard). The + /// early-stop readers (`get_anon`, `get_anon_mmap`, + /// `parse_lazy_index`) cannot make that check by design. pub fn get(buf: &mut &[u8]) -> Result { - // Header - let tag = Tag4::get(buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get: expected Env variant 0, got {}", - tag.size - )); - } - - // Canonical merkle root (fixed 32 bytes). For empty const sets the - // stored value is `zero_address()`. Verified against the - // recomputed value at the end of deserialization. - let stored_root = get_address(buf)?; + // Header: tag + stored merkle root (verified at the end against + // the recomputed root; empty const sets store `zero_address()`) + + // bundle fields. + let (stored_root, main, assumptions) = read_env_header(buf, "Env::get")?; #[cfg_attr(not(target_arch = "riscv64"), allow(unused_mut))] let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs - let num_blobs = get_u64(buf)?; - for _ in 0..num_blobs { - let addr = get_address(buf)?; - let len = get_u64(buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - *buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (hash-verified per entry) + for (addr, bytes) in read_blob_section(buf, "Env::get")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (lazy: read length prefix, slice bytes, defer parse) @@ -1587,7 +1759,19 @@ impl Env { .insert(addr, crate::lazy::LazyConstant::from_bytes(bytes.into())); } - // Section 3: Names (build lookup table and reverse index for metadata) + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!("Env::get: main {} not present in consts", m.hex())); + } + + // Section 3: anon_hints + for (addr, hints) in read_hints_section(buf)? { + env.anon_hints.insert(addr, hints); + } + + // Section 4: Names (build lookup table and reverse index for metadata) let num_names = get_u64(buf)?; let mut names_lookup: FxHashMap = FxHashMap::default(); let mut name_reverse_index: NameReverseIndex = @@ -1606,7 +1790,7 @@ impl Env { env.names.insert(addr, name); } - // Section 4: Named (use indexed deserialization for metadata) + // Section 5: Named (use indexed deserialization for metadata) let num_named = get_u64(buf)?; for _ in 0..num_named { let name_addr = get_address(buf)?; @@ -1617,7 +1801,7 @@ impl Env { env.named.insert(name, named); } - // Section 5: Comms + // Section 6: Comms let num_comms = get_u64(buf)?; for _ in 0..num_comms { let addr = get_address(buf)?; @@ -1625,9 +1809,6 @@ impl Env { env.comms.insert(addr, comm); } - // Optional trailing anon_hints section (see `Env::put`). - read_anon_hints_section(buf, &mut env)?; - // Verify the stored merkle root matches what we'd compute from // env.consts. Empty const set → expected = zero_address(). // Rejects any tampering with the header. @@ -1644,6 +1825,15 @@ impl Env { )); } + // Comms is the final section; anything after it is truncation + // damage or concatenated garbage. + if !buf.is_empty() { + return Err(format!( + "Env::get: {} trailing bytes after final section", + buf.len() + )); + } + Ok(env) } @@ -1664,41 +1854,15 @@ impl Env { pub fn parse_lazy_index(data: &[u8]) -> Result { let mut buf: &[u8] = data; - // Header: Tag4 (flag/variant) + canonical merkle root. - let tag = Tag4::get(&mut buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "parse_lazy_index: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "parse_lazy_index: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(&mut buf)?; + // Header: tag + merkle root + bundle fields. + let (stored_root, main, assumptions) = + read_env_header(&mut buf, "parse_lazy_index")?; - let mut index = LazyIndex::default(); + let mut index = LazyIndex { main, assumptions, ..LazyIndex::default() }; - // Section 1: Blobs (copied — small, and the kernel ingests their bytes). - let num_blobs = get_u64(&mut buf)?; - for _ in 0..num_blobs { - let addr = get_address(&mut buf)?; - let len = get_u64(&mut buf)? as usize; - if buf.len() < len { - return Err(format!( - "parse_lazy_index: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - buf = rest; - index.blobs.push((addr, bytes.to_vec())); - } + // Section 1: Blobs (copied — small, and the kernel ingests their + // bytes; hash-verified per entry). + index.blobs = read_blob_section(&mut buf, "parse_lazy_index")?; // Section 2: Consts — record offset windows, never parse the bodies. let num_consts = get_u64(&mut buf)?; @@ -1718,7 +1882,13 @@ impl Env { index.consts.push(LazyConstSlice { addr, offset, len }); } - // Section 3: Names — parsed to build the index for metadata decoding, then + // Section 3: anon_hints — the canonical hint channel (the writer + // always emits it, deriving from Named metadata when needed), so + // `LazyNamed.hint` is a plain lookup instead of a metadata harvest. + let hints_map: FxHashMap = + read_hints_section(&mut buf)?.into_iter().collect(); + + // Section 4: Names — parsed to build the index for metadata decoding, then // dropped (the check never consults the name component table). let num_names = get_u64(&mut buf)?; let mut names_lookup: FxHashMap = FxHashMap::default(); @@ -1733,8 +1903,9 @@ impl Env { names_lookup.insert(addr, name); } - // Section 4: Named — keep `name → addr` and the Defn reducibility hint; the - // full ConstantMeta (arena, CallSite, ...) is parsed then discarded. + // Section 5: Named — keep `name → addr` plus the §3 hint for that + // address; the full ConstantMeta (arena, CallSite, ...) is parsed + // then discarded. let num_named = get_u64(&mut buf)?; for _ in 0..num_named { let name_addr = get_address(&mut buf)?; @@ -1742,14 +1913,11 @@ impl Env { let name = names_lookup.get(&name_addr).cloned().ok_or_else(|| { format!("parse_lazy_index: missing name for addr {:?}", name_addr) })?; - let hint = match &named.meta().info { - super::metadata::ConstantMetaInfo::Def { hints, .. } => Some(*hints), - _ => None, - }; + let hint = hints_map.get(&named.addr).copied(); index.named.push(LazyNamed { name, addr: named.addr, hint }); } - // Section 5 (comms) is skipped: not needed by the check path. + // Section 6 (comms) is never reached: not needed by the check path. // Re-verify the merkle root over const addresses (header integrity). let mut const_addrs: Vec
= @@ -1765,63 +1933,50 @@ impl Env { )); } + // `main` must reference a constant present in the file. + if let Some(m) = &index.main + && const_addrs.binary_search(m).is_err() + { + return Err(format!( + "parse_lazy_index: main {} not present in consts", + m.hex() + )); + } + Ok(index) } - /// Anonymous-only deserialization: read the header + blobs + - /// consts sections, parse-and-discard the metadata sections - /// (names / named / comms). + /// Anonymous-only deserialization: read the header + §1 blobs + + /// §2 consts + §3 anon_hints, then STOP — the metadata sections + /// (§4 names / §5 named / §6 comms) are laid out after the hints + /// precisely so this reader never has to touch them. /// - /// Returns an `Env` with populated `consts` (lazy) and `blobs`, and - /// **empty** `named` / `names` / `comms`. The merkle-root header is - /// re-verified against the recomputed root over `consts.keys()`, - /// exactly as in [`Env::get`]. + /// Returns an `Env` with populated `consts` (lazy), `blobs`, and + /// `anon_hints`, and **empty** `named` / `names` / `comms`. The + /// merkle-root header is re-verified against the recomputed root + /// over `consts.keys()`, exactly as in [`Env::get`]. Because the + /// cursor stops at §3, trailing-buffer checks are impossible here + /// by design — only the full [`Env::get`] enforces EOF. /// - /// Why "parse and discard"? Sections 3-5 lack a section-level length - /// prefix today (only section 2 has one), so we can't byte-skip - /// them without parsing. Parsing into local scopes that drop on - /// return still wins us the steady-state memory: the returned `Env` - /// is metadata-free, and the temporary lookup tables / parsed - /// metadata values are reclaimed before this function returns. + /// Hints come straight from §3: the writer always emits that + /// section, deriving it from Named metadata when the in-memory map + /// is empty, so the historical read-time harvest from §Named is + /// gone. /// /// Used by the anon-mode kernel path so a verifier holding only - /// content addresses doesn't pay the long-term cost of metadata - /// sections it will never consult. + /// content addresses doesn't pay for metadata it will never consult. pub fn get_anon(buf: &mut &[u8]) -> Result { // Header (same as Env::get) - let tag = Tag4::get(buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get_anon: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get_anon: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(buf)?; + let (stored_root, main, assumptions) = + read_env_header(buf, "Env::get_anon")?; let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs (kept) - let num_blobs = get_u64(buf)?; - for _ in 0..num_blobs { - let addr = get_address(buf)?; - let len = get_u64(buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get_anon: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - *buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (kept; hash-verified per entry) + for (addr, bytes) in read_blob_section(buf, "Env::get_anon")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (kept, lazy) @@ -1851,55 +2006,27 @@ impl Env { .insert(addr, crate::lazy::LazyConstant::from_bytes(bytes.into())); } - // Section 3: Names — parse and DISCARD. We still need a populated - // `names_lookup` and `name_reverse_index` so section 4's indexed - // metadata parses correctly, but both go out of scope before - // returning so the steady-state `Env` carries no name data. - let num_names = get_u64(buf)?; - let mut names_lookup: FxHashMap = FxHashMap::default(); - let mut name_reverse_index: NameReverseIndex = - Vec::with_capacity(num_names as usize + 1); - let anon_addr = Address::from_blake3_hash(*Name::anon().get_hash()); - names_lookup.insert(anon_addr, Name::anon()); - for _ in 0..num_names { - let addr = get_address(buf)?; - let name = get_name_component(buf, &names_lookup)?; - name_reverse_index.push(addr.clone()); - names_lookup.insert(addr, name); - } - - // Section 4: Named — parse and mostly discard, but harvest - // `ReducibilityHints` from each `Def` variant into `env.anon_hints`. - // Hints are performance advice (lazy-delta tiebreak); the kernel's - // anon-mode correctness model is preserved either way. Without - // them, every Definition is forced to `Regular(0)` and the kernel - // can chew through `MAX_WHNF_FUEL` on definitions Lean would have - // marked `Abbrev`/`Regular(h)`. - let num_named = get_u64(buf)?; - for _ in 0..num_named { - let _name_addr = get_address(buf)?; - let named = get_named_indexed(buf, &name_reverse_index)?; - if let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &named.meta().info - { - env.anon_hints.insert(named.addr.clone(), *hints); - } + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!( + "Env::get_anon: main {} not present in consts", + m.hex() + )); } - // Section 5: Comms — parse and DISCARD. - let num_comms = get_u64(buf)?; - for _ in 0..num_comms { - let _addr = get_address(buf)?; - let _comm = Comm::get(buf)?; + // Section 3: anon_hints. Hints are performance advice (lazy-delta + // tiebreak); the kernel's anon-mode correctness model is preserved + // either way. Without them, every Definition is forced to + // `Regular(0)` and the kernel can chew through `MAX_WHNF_FUEL` on + // definitions Lean would have marked `Abbrev`/`Regular(h)`. + for (addr, hints) in read_hints_section(buf)? { + env.anon_hints.insert(addr, hints); } - // Optional trailing anon_hints section (see `Env::put`). For a closure - // sub-env this carries the hints the dropped Named section would have, so - // the kernel reproduces vanilla behavior with no def-eq overhead. - read_anon_hints_section(buf, &mut env)?; - - drop(names_lookup); - drop(name_reverse_index); + // Sections 4-6 (names / named / comms) are laid out after the + // hints precisely so this reader can stop here. // Verify merkle root over loaded consts. let mut const_addrs: Vec
= @@ -1929,11 +2056,10 @@ impl Env { /// the mapping stays alive as long as any consumer holds the env or /// any clone of a `LazyConstant` from it. /// - /// Sections 1 (blobs), 3 (names), 4 (named), and 5 (comms) are - /// handled the same way as `get_anon`: blobs are heap-copied (they - /// are small and consumed eagerly), names and named are - /// parse-and-discard (with hints harvested into `env.anon_hints`), - /// comms are skipped. + /// Sections are handled the same way as `get_anon`: §1 blobs are + /// heap-copied and hash-verified (small, eagerly consumed), §3 + /// anon_hints is read into `env.anon_hints`, and the reader stops + /// there — the metadata sections (§4-§6) are never touched. /// /// On Linux, the kernel's adaptive readahead handles the linear /// scan during section parsing efficiently; subsequent random @@ -1990,39 +2116,17 @@ impl Env { let mut buf: &[u8] = mmap_full; // Header (same shape as Env::get_anon) - let tag = Tag4::get(&mut buf)?; - if tag.flag != Self::FLAG { - return Err(format!( - "Env::get_anon_mmap: expected flag 0x{:X}, got 0x{:X}", - Self::FLAG, - tag.flag - )); - } - if tag.size != 0 { - return Err(format!( - "Env::get_anon_mmap: expected Env variant 0, got {}", - tag.size - )); - } - let stored_root = get_address(&mut buf)?; + let (stored_root, main, assumptions) = + read_env_header(&mut buf, "Env::get_anon_mmap")?; let mut env = Env::new(); + env.main = main; + env.assumptions = assumptions.into_iter().collect(); - // Section 1: Blobs (heap-copied; small, eagerly consumed) - let num_blobs = get_u64(&mut buf)?; - for _ in 0..num_blobs { - let addr = get_address(&mut buf)?; - let len = get_u64(&mut buf)? as usize; - if buf.len() < len { - return Err(format!( - "Env::get_anon_mmap: need {} bytes for blob, have {}", - len, - buf.len() - )); - } - let (bytes, rest) = buf.split_at(len); - buf = rest; - env.blobs.insert(addr, bytes.to_vec()); + // Section 1: Blobs (heap-copied; small, eagerly consumed; + // hash-verified per entry) + for (addr, bytes) in read_blob_section(&mut buf, "Env::get_anon_mmap")? { + env.blobs.insert(addr, bytes); } // Section 2: Consts (mmap-backed lazy windows) @@ -2060,47 +2164,23 @@ impl Env { buf = &buf[len..]; } - // Section 3: Names — parse and DISCARD (needed transiently so - // section 4's indexed metadata can be decoded). - let num_names = get_u64(&mut buf)?; - let mut names_lookup: FxHashMap = FxHashMap::default(); - let mut name_reverse_index: NameReverseIndex = - Vec::with_capacity(num_names as usize + 1); - let anon_addr = Address::from_blake3_hash(*Name::anon().get_hash()); - names_lookup.insert(anon_addr, Name::anon()); - for _ in 0..num_names { - let addr = get_address(&mut buf)?; - let name = get_name_component(&mut buf, &names_lookup)?; - name_reverse_index.push(addr.clone()); - names_lookup.insert(addr, name); - } - - // Section 4: Named — harvest `ReducibilityHints` from `Def` - // entries into `env.anon_hints`; discard the rest. See `get_anon` - // for the rationale. - let num_named = get_u64(&mut buf)?; - for _ in 0..num_named { - let _name_addr = get_address(&mut buf)?; - let named = get_named_indexed(&mut buf, &name_reverse_index)?; - if let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &named.meta().info - { - env.anon_hints.insert(named.addr.clone(), *hints); - } + // `main` must reference a constant actually present in the file. + if let Some(m) = &env.main + && env.consts.get(m).is_none() + { + return Err(format!( + "Env::get_anon_mmap: main {} not present in consts", + m.hex() + )); } - // Section 5: Comms — parse and DISCARD. - let num_comms = get_u64(&mut buf)?; - for _ in 0..num_comms { - let _addr = get_address(&mut buf)?; - let _comm = Comm::get(&mut buf)?; + // Section 3: anon_hints — see `get_anon` for the rationale. + for (addr, hints) in read_hints_section(&mut buf)? { + env.anon_hints.insert(addr, hints); } - // Optional trailing anon_hints section (see `Env::put`). - read_anon_hints_section(&mut buf, &mut env)?; - - drop(names_lookup); - drop(name_reverse_index); + // Sections 4-6 (names / named / comms) are laid out after the + // hints precisely so this reader can stop here. // Verify merkle root over loaded consts (same as get_anon). let mut const_addrs: Vec
= @@ -2126,20 +2206,29 @@ impl Env { Ok(buf.len()) } - /// Calculate serialized size with breakdown by section. + /// Calculate serialized size with breakdown by section: + /// `(header, blobs, consts, anon_hints, names, named, comms)`. pub fn serialized_size_breakdown( &self, - ) -> Result<(usize, usize, usize, usize, usize, usize), String> { + ) -> Result<(usize, usize, usize, usize, usize, usize, usize), String> { let mut buf = Vec::new(); - // Header + merkle root (matches Env::put layout; root is always - // 32 bytes, with `zero_address()` as the empty-env sentinel). + // Header: tag + merkle root (32 bytes, `zero_address()` sentinel + // for empty const sets) + bundle fields (matches Env::put layout). Tag4::new(Self::FLAG, 0).put(&mut buf); let mut const_addrs: Vec
= self.consts.iter().map(|e| e.key().clone()).collect(); const_addrs.sort_unstable(); let root = merkle_root_canonical(&const_addrs).unwrap_or_else(zero_address); put_address(&root, &mut buf); + put_opt_addr(&self.main, &mut buf); + let mut assumption_addrs: Vec
= + self.assumptions.iter().cloned().collect(); + assumption_addrs.sort_unstable(); + put_u64(assumption_addrs.len() as u64, &mut buf); + for addr in &assumption_addrs { + put_address(addr, &mut buf); + } let header_size = buf.len(); // Section 1: Blobs @@ -2162,7 +2251,32 @@ impl Env { } let consts_size = buf.len() - before_consts; - // Section 3: Names (also build name index) + // Section 3: anon_hints (mirrors Env::put's derive-from-named rule) + let before_hints = buf.len(); + let mut hint_pairs: Vec<(Address, ReducibilityHints)> = + if self.anon_hints.is_empty() { + let mut derived: FxHashMap = + FxHashMap::default(); + for entry in self.named.iter() { + if let super::metadata::ConstantMetaInfo::Def { hints, .. } = + &entry.value().meta().info + { + derived.entry(entry.value().addr.clone()).or_insert(*hints); + } + } + derived.into_iter().collect() + } else { + self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() + }; + hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + put_u64(hint_pairs.len() as u64, &mut buf); + for (addr, hints) in &hint_pairs { + put_address(addr, &mut buf); + hints.put_ser(&mut buf); + } + let hints_size = buf.len() - before_hints; + + // Section 4: Names (also build name index) let before_names = buf.len(); let sorted_names = topological_sort_names(&self.names); let mut name_index: NameIndex = NameIndex::new(); @@ -2174,7 +2288,7 @@ impl Env { } let names_size = buf.len() - before_names; - // Section 4: Named (use indexed serialization) + // Section 5: Named (use indexed serialization) let before_named = buf.len(); put_u64(self.named.len() as u64, &mut buf); for entry in self.named.iter() { @@ -2183,7 +2297,7 @@ impl Env { } let named_size = buf.len() - before_named; - // Section 5: Comms + // Section 6: Comms let before_comms = buf.len(); put_u64(self.comms.len() as u64, &mut buf); for entry in self.comms.iter() { @@ -2196,6 +2310,7 @@ impl Env { header_size, blobs_size, consts_size, + hints_size, names_size, named_size, comms_size, @@ -2385,7 +2500,7 @@ mod tests { } fn gen_env(g: &mut Gen) -> Env { - let env = Env::new(); + let mut env = Env::new(); // Generate blobs let num_blobs = gen_range(g, 0..10); @@ -2407,12 +2522,14 @@ mod tests { // Generate constants and named entries let num_consts = gen_range(g, 0..10); + let mut const_addrs: Vec
= Vec::new(); for i in 0..num_consts { let constant = gen_constant(g); let mut buf = Vec::new(); constant.put(&mut buf); let addr = Address::hash(&buf); env.store_const(addr.clone(), constant); + const_addrs.push(addr.clone()); // Create a named entry for this constant if !names.is_empty() { @@ -2448,6 +2565,34 @@ mod tests { env.comms.insert(addr, comm); } + // Bundle fields. `main` must reference a stored const — `Env::put` + // validates that; `assumptions` are opaque addresses (no content + // behind them is required), so random ones exercise the encoding. + if !const_addrs.is_empty() && bool::arbitrary(g) { + let idx = usize::arbitrary(g) % const_addrs.len(); + env.main = Some(const_addrs[idx].clone()); + } + let num_assumptions = gen_range(g, 0..5); + for _ in 0..num_assumptions { + env.assumptions.insert(Address::arbitrary(g)); + } + + // Explicit anon_hints (keyed by arbitrary addresses — the map is + // advisory). When left empty, `Env::put` derives §3 from Named + // metadata instead; gen_env metas are all `Empty`, so that + // derivation contributes nothing and roundtrip equality holds + // either way. + let num_hints = gen_range(g, 0..4); + for _ in 0..num_hints { + let variant: u8 = Arbitrary::arbitrary(g); + let hint = match variant % 3 { + 0 => ReducibilityHints::Opaque, + 1 => ReducibilityHints::Abbrev, + _ => ReducibilityHints::Regular(Arbitrary::arbitrary(g)), + }; + env.anon_hints.insert(Address::arbitrary(g), hint); + } + env } @@ -2546,6 +2691,29 @@ mod tests { } } + // Bundle fields + hints (gen_env metas are all Empty, so §3 + // derivation contributes nothing and plain equality holds). + if env.main != recovered.main { + eprintln!("main mismatch: {:?} vs {:?}", env.main, recovered.main); + return false; + } + if env.assumptions != recovered.assumptions { + eprintln!( + "assumptions mismatch: {} vs {} entries", + env.assumptions.len(), + recovered.assumptions.len() + ); + return false; + } + if env.anon_hints != recovered.anon_hints { + eprintln!( + "anon_hints mismatch: {} vs {} entries", + env.anon_hints.len(), + recovered.anon_hints.len() + ); + return false; + } + true }, Err(e) => { @@ -2683,21 +2851,39 @@ mod tests { assert!(res.is_err(), "tampered root should be rejected"); } + /// Byte offset of the first constant's payload within a serialized + /// env buffer, computed by walking the header + §1 (blobs) + the §2 + /// prefix with the real parsers — so tampering tests track layout + /// changes instead of hardcoding offsets. + fn first_const_payload_offset(buf: &[u8]) -> usize { + let mut cur: &[u8] = buf; + read_env_header(&mut cur, "test").unwrap(); + read_blob_section(&mut cur, "test").unwrap(); + let n = get_u64(&mut cur).unwrap(); + assert!(n >= 1, "need at least one const to locate"); + let _addr = get_address(&mut cur).unwrap(); + let _len = Tag0::get(&mut cur).unwrap(); + buf.len() - cur.len() + } + + /// Byte offset of the first blob's payload (same technique as + /// [`first_const_payload_offset`]). + fn first_blob_payload_offset(buf: &[u8]) -> usize { + let mut cur: &[u8] = buf; + read_env_header(&mut cur, "test").unwrap(); + let n = get_u64(&mut cur).unwrap(); + assert!(n >= 1, "need at least one blob to locate"); + let _addr = get_address(&mut cur).unwrap(); + let _len = get_u64(&mut cur).unwrap(); + buf.len() - cur.len() + } + /// Flip a byte inside the first const's payload bytes (not its /// stored address): merkle still validates over `consts.keys()`, so /// the per-entry `Address::hash(bytes) == addr` check is what must /// reject this corruption. Without that check, `Env::get` would /// succeed and the failure would surface much later inside /// `LazyConstant::get` with a misleading parse error. - /// - /// Header layout for an env with empty blobs and one const: - /// [0] Tag4 (0xE0) - /// [1..33) merkle root (32 bytes) - /// [33] Section 1 (blobs) count = 0 (Tag0) - /// [34] Section 2 (consts) count = 1 (Tag0) - /// [35..67) const address (32 bytes) - /// [67] Tag0 length of const bytes - /// [68..] const bytes (target for tampering) #[test] fn env_const_bytes_tampering_rejected_by_get() { let env = Env::new(); @@ -2705,7 +2891,7 @@ mod tests { let mut buf = Vec::new(); env.put(&mut buf).unwrap(); // Flip a byte well inside the const payload. - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len(), "expected const bytes at offset {off}"); buf[off] ^= 0xFF; let res = Env::get(&mut buf.as_slice()); @@ -2722,7 +2908,7 @@ mod tests { env.store_const(Address::hash(b"a"), defn_const(vec![])); let mut buf = Vec::new(); env.put(&mut buf).unwrap(); - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len()); buf[off] ^= 0xFF; let res = Env::get_anon(&mut buf.as_slice()); @@ -2740,7 +2926,7 @@ mod tests { env.store_const(Address::hash(b"a"), defn_const(vec![])); let mut buf = Vec::new(); env.put(&mut buf).unwrap(); - let off = 68 + 3; + let off = first_const_payload_offset(&buf) + 3; assert!(off < buf.len()); buf[off] ^= 0xFF; // mmap requires a real file @@ -2885,4 +3071,187 @@ mod tests { assert_eq!(pre_a, post_a, "`a` content should be stable across unlink"); assert_ne!(post_a, post_b, "discriminators should still differentiate"); } + + // --------------------------------------------------------------------------- + // Bundle header fields (main / assumptions) + §1/§3 integrity + // --------------------------------------------------------------------------- + + #[test] + fn env_main_and_assumptions_roundtrip_all_readers() { + let mut env = Env::new(); + let a = store_canonical(&env, defn_const(vec![])); + store_canonical(&env, defn_const_discriminator(vec![], 1)); + env.main = Some(a.clone()); + env.assumptions.insert(Address::hash(b"assume-1")); + env.assumptions.insert(Address::hash(b"assume-2")); + + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + + let full = Env::get(&mut buf.as_slice()).unwrap(); + assert_eq!(full.main, Some(a.clone())); + assert_eq!(full.assumptions, env.assumptions); + + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!(anon.main, Some(a.clone())); + assert_eq!(anon.assumptions, env.assumptions); + + let index = Env::parse_lazy_index(&buf).unwrap(); + assert_eq!(index.main, Some(a)); + let mut sorted: Vec
= env.assumptions.iter().cloned().collect(); + sorted.sort_unstable(); + assert_eq!(index.assumptions, sorted, "LazyIndex keeps header order"); + } + + #[test] + fn env_put_rejects_main_not_in_consts() { + let mut env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.main = Some(Address::hash(b"not-a-const")); + let mut buf = Vec::new(); + let err = env + .put(&mut buf) + .expect_err("main outside consts must be rejected at write time"); + assert!(err.contains("main"), "got: {err}"); + } + + /// Flip a byte of the serialized `main` address: the header parses, + /// but the resulting address is (w.h.p.) not a stored constant, so + /// the `main ∈ consts` reader check must reject the file. This test + /// intentionally knows the fixed header prefix (tag byte + 32-byte + /// root — same knowledge as `parse_stored_root`). + #[test] + fn env_get_rejects_tampered_main() { + let mut env = Env::new(); + let a = store_canonical(&env, defn_const(vec![])); + env.main = Some(a); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + assert_eq!(buf[33], 0x01, "main opt-flag should be Some"); + buf[34] ^= 0xFF; + let err = Env::get(&mut buf.as_slice()) + .expect_err("tampered main must be rejected"); + assert!(err.contains("main"), "got: {err}"); + } + + #[test] + fn env_get_rejects_unsorted_assumptions() { + let mut env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.assumptions.insert(Address::hash(b"x")); + env.assumptions.insert(Address::hash(b"y")); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + // Locate the header end with the real parser, then swap the two + // 32-byte assumption addresses in place (the writer sorts them). + let header_len = { + let mut cur: &[u8] = &buf; + read_env_header(&mut cur, "test").unwrap(); + buf.len() - cur.len() + }; + let (lo, hi) = (header_len - 64, header_len - 32); + let first: Vec = buf[lo..hi].to_vec(); + let second: Vec = buf[hi..header_len].to_vec(); + buf[lo..hi].copy_from_slice(&second); + buf[hi..header_len].copy_from_slice(&first); + let err = Env::get(&mut buf.as_slice()) + .expect_err("descending assumptions must be rejected"); + assert!(err.contains("strictly ascending"), "got: {err}"); + } + + #[test] + fn env_blob_tampering_rejected_by_all_readers() { + use std::io::Write; + let env = Env::new(); + store_canonical(&env, defn_const(vec![])); + env.store_blob(b"blob payload".to_vec()); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + let off = first_blob_payload_offset(&buf) + 2; + buf[off] ^= 0xFF; + + for (reader, res) in [ + ("get", Env::get(&mut buf.as_slice()).err()), + ("get_anon", Env::get_anon(&mut buf.as_slice()).err()), + ("parse_lazy_index", Env::parse_lazy_index(&buf).err()), + ] { + let err = + res.unwrap_or_else(|| panic!("{reader}: tampered blob accepted")); + assert!( + err.contains("blob at idx"), + "{reader}: expected blob verify error, got: {err}" + ); + } + + let tmp = std::env::temp_dir().join("ix_env_blob_tamper_mmap_test.ixe"); + { + let mut f = std::fs::File::create(&tmp).unwrap(); + f.write_all(&buf).unwrap(); + } + let err = Env::get_anon_mmap(&tmp) + .expect_err("get_anon_mmap: tampered blob accepted"); + assert!(err.contains("blob at idx"), "got: {err}"); + std::fs::remove_file(&tmp).ok(); + } + + #[test] + fn env_get_rejects_trailing_garbage_but_get_anon_stops_early() { + let env = Env::new(); + store_canonical(&env, defn_const(vec![])); + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + buf.extend_from_slice(&[0xAB, 0xCD, 0xEF]); + let err = Env::get(&mut buf.as_slice()) + .expect_err("trailing bytes must be rejected by the full reader"); + assert!(err.contains("trailing"), "got: {err}"); + // get_anon stops after §3 and never sees the tail — by design. + Env::get_anon(&mut buf.as_slice()) + .expect("get_anon early-stops before the garbage"); + } + + /// §3 is the canonical hint channel: with an empty `anon_hints` map + /// the writer derives the section from Named `Def` metadata, and an + /// env carrying the same hints explicitly serializes byte-identically. + #[test] + fn env_hints_derived_from_named_metadata() { + use crate::env::Named; + use crate::metadata::{ConstantMetaInfo, ExprMeta}; + + let env = Env::new(); + let const_addr = store_canonical(&env, defn_const(vec![])); + let name = Name::str(Name::anon(), "hinted".to_string()); + let name_addr = Address::from_blake3_hash(*name.get_hash()); + env.names.insert(name_addr.clone(), name.clone()); + let meta = ConstantMeta::new(ConstantMetaInfo::Def { + name: name_addr, + lvls: vec![], + hints: ReducibilityHints::Regular(7), + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }); + env.named.insert(name, Named::new(const_addr.clone(), meta)); + + // Empty map → §3 derived from Named at write time; the anon + // reader picks it up without ever parsing the Named section. + let mut buf = Vec::new(); + env.put(&mut buf).unwrap(); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!( + anon.anon_hints.get(&const_addr), + Some(&ReducibilityHints::Regular(7)) + ); + + // Explicit map with identical content → identical bytes. + let mut env2 = env.clone(); + env2.anon_hints.insert(const_addr, ReducibilityHints::Regular(7)); + let mut buf2 = Vec::new(); + env2.put(&mut buf2).unwrap(); + assert_eq!( + buf, buf2, + "derived and explicit hint sections should serialize identically" + ); + } } diff --git a/crates/ixvm-codegen/src/env_handle.rs b/crates/ixvm-codegen/src/env_handle.rs index f2c0354e0..1e1f6e3d8 100644 --- a/crates/ixvm-codegen/src/env_handle.rs +++ b/crates/ixvm-codegen/src/env_handle.rs @@ -6,12 +6,12 @@ //! per-shard FFI calls take a `&EnvHandle` (shared across the batch) so //! the env is parsed exactly once instead of every call. //! -//! `anon_hints` (`Defn` reducibility hints) are populated at handle -//! construction time. `Env::get_anon_mmap` already harvests them; -//! `Env::get` (full-form decode used by the bytes-blob path) does -//! not, so `from_bytes` runs the harvest post-decode. +//! `anon_hints` (`Defn` reducibility hints) come from the `.ixe` §3 +//! hints section, which every writer emits (deriving it from Named +//! metadata when the in-memory map is empty) — both readers used here +//! populate the map directly, so no post-decode harvest is needed. -use ixon::{Env, metadata::ConstantMetaInfo}; +use ixon::Env; pub struct EnvHandle { pub env: Env, @@ -19,38 +19,17 @@ pub struct EnvHandle { impl EnvHandle { /// Load via `Env::get_anon_mmap` (zero-copy mmap of the `.ixe` file). - /// Anon-mode parser already harvests `anon_hints` — no post-pass. pub fn from_ixe_path(path: &std::path::Path) -> Result { let env = Env::get_anon_mmap(path)?; Ok(Self { env }) } /// Decode a serialized env blob (`Ixon.serEnv` output) via - /// `Env::get`, then harvest `anon_hints` from each `Def` named - /// entry. Used by the compiled-Lean-env path where the env is built - /// in Lean memory and serialized for the cross-FFI handoff. + /// `Env::get`. Used by the compiled-Lean-env path where the env is + /// built in Lean memory and serialized for the cross-FFI handoff. pub fn from_bytes(bytes: &[u8]) -> Result { let mut cursor: &[u8] = bytes; - let mut env = Env::get(&mut cursor)?; - // `Env::get` reads named entries but doesn't populate - // `env.anon_hints`. Walk `env.named` and insert each `Def` - // variant's `hints`. Mirrors the in-line harvest inside - // `Env::get_anon`. - let hints: Vec<_> = env - .named - .iter() - .filter_map(|entry| { - let named = entry.value(); - if let ConstantMetaInfo::Def { hints, .. } = &named.meta().info { - Some((named.addr.clone(), *hints)) - } else { - None - } - }) - .collect(); - for (addr, h) in hints { - env.anon_hints.insert(addr, h); - } + let env = Env::get(&mut cursor)?; Ok(Self { env }) } } diff --git a/crates/kernel/src/anon_work.rs b/crates/kernel/src/anon_work.rs index 1f5be74dd..5ffe10377 100644 --- a/crates/kernel/src/anon_work.rs +++ b/crates/kernel/src/anon_work.rs @@ -193,70 +193,11 @@ pub fn closure_addrs( source: &IxonEnv, roots: &[Address], ) -> std::collections::HashSet
{ - use std::collections::{HashSet, VecDeque}; - - use ixon::constant::{ConstantInfo as CI, MutConst as MC}; - - let proj_block = |info: &CI| -> Option
{ - match info { - CI::IPrj(p) => Some(p.block.clone()), - CI::CPrj(p) => Some(p.block.clone()), - CI::RPrj(p) => Some(p.block.clone()), - CI::DPrj(p) => Some(p.block.clone()), - _ => None, - } - }; - - let mut closure: HashSet
= HashSet::default(); - let mut queue: VecDeque
= VecDeque::new(); - let push = - |closure: &mut HashSet
, q: &mut VecDeque
, a: Address| { - if closure.insert(a.clone()) { - q.push_back(a); - } - }; - for r in roots { - push(&mut closure, &mut queue, r.clone()); - } - while let Some(addr) = queue.pop_front() { - if let Some(c) = source.get_const(&addr) { - // 1. Expr-level refs. - for r in &c.refs { - push(&mut closure, &mut queue, r.clone()); - } - // 2. A projection → its Muts block (structural, not in `refs`). - if let Some(b) = proj_block(&c.info) { - push(&mut closure, &mut queue, b); - } - // 3. A Muts block → ALL its member + constructor projection entries. - // Ingressing a block (`ingress_anon_block`) computes these projection - // addresses and requires them present, even when nothing references them - // directly via `refs`. Mirror `build_anon_work`'s enumeration so the - // sub-env carries them (else the guest fails with "computed CPrj address - // … not present in env"). - if let CI::Muts(members) = &c.info { - for (i, m) in members.iter().enumerate() { - let i = i as u64; - let member_addr = match m { - MC::Defn(_) => anon_defn_proj_addr(&addr, i), - MC::Indc(_) => anon_indc_proj_addr(&addr, i), - MC::Recr(_) => anon_recr_proj_addr(&addr, i), - }; - push(&mut closure, &mut queue, member_addr); - if let MC::Indc(ind) = m { - for cidx in 0..ind.ctors.len() as u64 { - push( - &mut closure, - &mut queue, - anon_ctor_proj_addr(&addr, i, cidx), - ); - } - } - } - } - } - } - closure + // The traversal lives in `ixon` (`Env::bfs_closure`) so bundle + // pruning/validation and the shard sub-env share one edge + // definition; this wrapper keeps the kernel-side std-HashSet + // signature. + source.bfs_closure(roots).into_iter().collect() } /// Build a closure sub-env: serialize only the BFS dependency closure of @@ -265,9 +206,9 @@ pub fn closure_addrs( /// per-const integrity check (`hash(bytes) == addr`) and the env merkle root /// still hold. The guest decodes this instead of the whole env, so it pays only /// its closure's decode — essential for envs that don't fit the guest whole -/// (Init, 184 MB doesn't fit the 512 MB Zisk guest). Empty `anon_hints` (no -/// metadata section) is performance-only: ingress falls back to `Regular(0)`, -/// so the typecheck result — and thus the committed claim — is unchanged. +/// (Init, 184 MB doesn't fit the 512 MB Zisk guest). Missing hints are +/// performance-only: ingress falls back to `Regular(0)`, so the typecheck +/// result — and thus the committed claim — is unchanged. /// External refs (not in `source`) are omitted and remain open assumptions, /// exactly as in whole-env. /// @@ -300,9 +241,10 @@ fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { } // else: external ref absent from `source` — omit; stays an open assumption. // Carry the constant's reducibility hint so the guest reproduces vanilla - // kernel behavior. `get_anon` normally harvests hints from the Named - // section, which this sub-env drops; without them the kernel forces - // `Regular(0)` and does extra def-eq work (the ~30% check overhead). + // kernel behavior. The sub-env has no Named section to derive the §3 + // hints from at serialization time, so populate the map explicitly; + // without hints the kernel forces `Regular(0)` and does extra def-eq + // work (the ~30% check overhead). if let Some(h) = source.anon_hints.get(addr) { sub.anon_hints.insert(addr.clone(), *h); } diff --git a/crates/kernel/src/ingress.rs b/crates/kernel/src/ingress.rs index b7349bd0b..ad6660daa 100644 --- a/crates/kernel/src/ingress.rs +++ b/crates/kernel/src/ingress.rs @@ -3684,11 +3684,21 @@ fn drop_ixon_env(_ixon_env: IxonEnv, _quiet: bool) { fn drop_ixon_env(ixon_env: IxonEnv, quiet: bool) { let total_start = Instant::now(); // `anon_hints` is a small FxHashMap (one entry per Def from the .ixe's - // Named metadata); dropping it inline alongside the bookkeeping below - // is negligible compared to the DashMap dropdance. + // hints section); `main`/`assumptions` are a single address and a small + // set. Dropping them inline alongside the bookkeeping below is + // negligible compared to the DashMap dropdance. // `..` covers the env's private fields. - let IxonEnv { consts, named, blobs, names, comms, anon_hints: _, .. } = - ixon_env; + let IxonEnv { + consts, + named, + blobs, + names, + comms, + anon_hints: _, + main: _, + assumptions: _, + .. + } = ixon_env; let consts_len = consts.len(); let named_len = named.len(); let names_len = names.len(); diff --git a/docs/Ixon.md b/docs/Ixon.md index f8a2798e8..70418060f 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -802,15 +802,22 @@ plus `.ixe`. See `src/ix/ixon/serialize.rs::Env::put` for the byte-level layout. The .ixe layout is a Tag4(0xE, 0) header byte followed by a 32-byte -canonical merkle root and then 5 sections: +canonical merkle root, the bundle header fields, and then 6 sections +(hot data first, metadata last): ``` -Header: Tag4 { flag: 0xE, size: 0 } -- one byte (0xE0) -Root: 32 bytes -- canonical merkle root over - consts.keys(); for empty - const sets this is the - fixed `zero_address` - sentinel. +Header: Tag4 { flag: 0xE, size: 0 } -- one byte (0xE0) +Root: 32 bytes -- canonical merkle root over + consts.keys(); for empty + const sets this is the + fixed `zero_address` + sentinel. +Main: 1 byte (0x00 | 0x01) + 32 bytes if 0x01 + -- optional bundle root; see + "Bundles" below. +Assumptions: count (Tag0) + [Address (32 bytes)]* + -- strictly ascending; the + bundle trust boundary. ``` The root is mandatory (non-optional): every env has a unique canonical @@ -819,12 +826,23 @@ produce byte-identical roots regardless of construction order. Deserialization recomputes the root from `consts` and rejects any mismatch as tampered. +`main` and `assumptions` are NOT covered by the consts merkle root — +`main` is a convenience pointer (readers verify `main ∈ consts`; +consumers holding an externally-expected address must compare), and +the assumptions root, when a claim needs it, is recomputed as +`merkle_root_canonical(leaves)` from the section (identical to the +root `AssumptionTree::canonical` produces over the same leaves). + **Section 1: Blobs** (Address → raw bytes) ``` count (Tag0) [Address (32 bytes) + len (Tag0) + bytes]* ``` +Every reader verifies `blake3(bytes) == addr` per blob entry — a +swapped blob would otherwise silently change a Nat/String literal's +value under an otherwise-valid file. + **Section 2: Constants** (Address → length-prefixed Constant bytes) ``` count (Tag0) @@ -833,42 +851,85 @@ count (Tag0) The Tag0 length sidecar is a **section-level** framing byte: it is not part of the constant's content-addressed bytes. The address is -computed as `blake3` over only the Tag4 constant body. This layout -lets a lazy loader slice each constant directly into a -[`LazyConstant`](../src/ix/ixon/lazy.rs) without parsing its Tag4 -envelope, deferring full deserialization until first access. The -materialized `Constant` is cached so subsequent accesses are free. - -### Anonymous-only loading +computed as `blake3` over only the Tag4 constant body (verified per +entry on load). This layout lets a lazy loader slice each constant +directly into a [`LazyConstant`](../crates/ixon/src/lazy.rs) without +parsing its Tag4 envelope, deferring full deserialization until first +access. -`Env::get_anon` (`src/ix/ixon/serialize.rs`) is a sibling of -`Env::get` that loads only the anonymous sections — header, blobs, -consts — and parses-and-drops the metadata sections (names, named, -comms). The returned `Env` has empty `named`/`names`/`comms` and is -suitable for anon-mode kernel workflows that never consult metadata. -Steady-state memory for a Mathlib-scale env drops from ~3-4 GB -(structured + metadata) to ~1 GB (lazy bytes only). +**Section 3: Reducibility hints** (Address → ReducibilityHints) +``` +count (Tag0) +[Address (32 bytes) + ReducibilityHints]* +``` -Exposed to Lean via `rs_de_env_anon` (`Ix.Ixon.rsDeEnvAnon`). +The canonical hint channel for the anon/lazy readers, keyed by +constant address and sorted ascending. Writers ALWAYS emit this +section: when the in-memory hint map is empty (the compile path — +hints live in Named metadata), the section is derived from the Named +entries' `Def` metadata at write time. Hints are performance-only +advice (the `Regular(0)` fallback is always correct) and are +intentionally outside the consts merkle root. -**Section 3: Names** (Address → NameComponent, topologically sorted) +**Section 4: Names** (Address → NameComponent, topologically sorted) ``` count (Tag0) [Address (32 bytes) + NameComponent]* ``` -**Section 4: Named** (Name Address → Named with indexed metadata) +**Section 5: Named** (Name Address → Named with indexed metadata) ``` count (Tag0) [NameAddress (32 bytes) + ConstAddress (32 bytes) + ConstantMeta]* ``` -**Section 5: Commitments** (Address → Comm) +**Section 6: Commitments** (Address → Comm) ``` count (Tag0) [Address (32 bytes) + secret_addr (32 bytes) + payload_addr (32 bytes)]* ``` +The full reader (`Env::get` / Lean `getEnv`) rejects trailing bytes +after Section 6. + +### Anonymous-only loading + +`Env::get_anon` (`crates/ixon/src/serialize.rs`) is a sibling of +`Env::get` that reads the header, §1 blobs, §2 consts, and §3 hints, +then STOPS — the metadata sections are laid out after the hints +precisely so anon readers never touch them. The returned `Env` has +empty `named`/`names`/`comms` and is suitable for anon-mode kernel +workflows that never consult metadata. Steady-state memory for a +Mathlib-scale env drops from ~3-4 GB (structured + metadata) to ~1 GB +(lazy bytes only). + +Exposed to Lean via `rs_de_env_anon` (`Ix.Ixon.rsDeEnvAnon`); +`Env::get_anon_mmap` is the zero-copy mmap sibling and +`Env::parse_lazy_index` the zero-copy index variant (reads through §5, +skipping only comms). + +### Bundles: pinning a single value + +A **bundle** is an `.ixe` whose `main` points at a distinguished +constant (e.g. an anonymous `Defn` wrapping a value, produced by +`Ix.Commit.compileDef`) and whose contents are closed up to +`assumptions`. Because a constant's address is a merkle root over its +entire dependency DAG (refs tables hold addresses of constants and +literal blobs, recursively), `main`'s 32 bytes alone pin the value; +the bundle is the data-availability artifact that ships the bytes +behind those addresses. + +- `Env::prune_to_closure(main, assumed)` builds a bundle: the 3-edge + closure of `main` (Expr refs; projection → `Muts` block; `Muts` + block → member/constructor projections), cut at `assumed`, carrying + constants (genuine bytes), blobs, per-constant hints, and display + metadata (named entries, name components + string blobs, `DataValue` + payload blobs, `meta_refs` extension edges, aux_gen originals). +- `Env::validate_closed()` is the receiver-side check: `main ∈ consts` + and every reachable address is carried (consts ∪ blobs) or assumed. +- Whole-environment files are the degenerate case (`main` absent, + `assumptions` empty). + --- ## Proofs and Claims @@ -881,7 +942,7 @@ fits in single-byte tags (sizes 0..=7 per flag). | Size | Byte | Type | Payload | |------|------|------|---------| -| 0 | `0xE0` | Environment | bare 32-byte merkle root + 5 sections | +| 0 | `0xE0` | Environment | 32-byte merkle root + main/assumptions + 6 sections | | 1 | `0xE1` | Commitment | 2 addr: secret, payload | | 2 | `0xE2` | AssumptionTree | recursive merkle-tree body (see below) | | 3 | `0xE3` | Eval claim | 2 addr (input, output) + opt assumptions | From 07a914a094c0acdfcabac0f37d6d915a84e3c87e Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Thu, 9 Jul 2026 21:29:31 -0400 Subject: [PATCH 2/8] =?UTF-8?q?Ixon:=20ix=20diff=20=E2=80=94=20structured?= =?UTF-8?q?=20env=20diff=20(Rust=20engine,=20FFI=20report,=20Lean=20CLI)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lake exe ix diff [--anon|--meta] [--verbose] compares two serialized environments and exits 0/1/2 (GNU diff convention). Engine (crates/ixon/src/diff.rs, host-gated like prune_to_closure): - diff_envs(a, b, meta): named join on Name (added/removed/changed/ meta-only), consts/blobs set diffs, comms join, main/assumptions, hints joined on shared consts. Deterministic ordering. - exprs_equal: iterative lockstep walk resolving Share/Sort/Ref/Str/ Nat/Prj indices through each side's own Constant tables (same bytes + different tables => different; permuted tables with equal resolution => equal), pointer-pair memoized, cycle-guarded. - Per-kind field classification (type/value/lvls/safety/rules[i].rhs/ ctors[j].*, ...); kind changes label "kind"; representation-only address churn labels "encoding". Projections descend one level into their Muts block: changed member => block.* labels, untouched member => "block-siblings". - Default anon mode compares only anonymous structure (hints included: they live in anon §3); meta mode also compares ConstantMeta/original per component. FFI: rs_diff_envs (ByteArray -> ByteArray -> Bool -> Except String EnvDiff) parses both inputs with the full reader (Named.original participates) and marshals EnvStats/NamedDiff/EnvDiff (lean.rs layouts num_64:4 / num_obj:7 / num_obj:17); names cross pre-rendered and pre-sorted, addresses raw. Tests: 15 engine unit tests (refs-shift guard, share-vs-inline, univ-table shift, memo-clearing regression, block descent, ...) and 12 Lean FFI tests incl. self-diff-empty properties over genRawEnv in both modes. --- Ix/Cli/DiffCmd.lean | 190 +++++ Ix/Ixon.lean | 85 ++ Main.lean | 2 + Tests/FFI/Ixon.lean | 103 +++ crates/ffi/src/lean.rs | 10 + crates/ffi/src/lean_ixon.rs | 1 + crates/ffi/src/lean_ixon/diff.rs | 165 ++++ crates/ixon/src/diff.rs | 1375 ++++++++++++++++++++++++++++++ crates/ixon/src/lib.rs | 4 + 9 files changed, 1935 insertions(+) create mode 100644 Ix/Cli/DiffCmd.lean create mode 100644 crates/ffi/src/lean_ixon/diff.rs create mode 100644 crates/ixon/src/diff.rs diff --git a/Ix/Cli/DiffCmd.lean b/Ix/Cli/DiffCmd.lean new file mode 100644 index 000000000..888ae1bd1 --- /dev/null +++ b/Ix/Cli/DiffCmd.lean @@ -0,0 +1,190 @@ +/- + `ix diff `: structured diff of two serialized Ixon + environments. + + The diff itself is computed in Rust (`rs_diff_envs` → `ixon::diff`): + both files are parsed with the full reader and compared on anonymous + structure — names serve as join/display keys, constants compare by + content address with per-field classification (type/value/lvls/…, + `block.*` for projection targets, `"encoding"` when only the + representation moved). `--meta` additionally compares named metadata + (`ConstantMeta`/`original`). + + Exit codes (GNU diff convention): 0 = no difference found in the + selected mode, 1 = differences found, 2 = error. +-/ +module +public import Cli +public import Ix.Address +public import Ix.Common +public import Ix.Ixon + +public section + +namespace Ix.Cli.DiffCmd + +private def pad (s : String) (w : Nat) : String := + s.pushn ' ' (w - s.length) + +private def shortAddr (verbose : Bool) (a : Address) : String := + if verbose then toString a else ((toString a).take 12).toString ++ "…" + +/-- Synthetic mutual-block names embed the block hash as their second + component (`Ix.<64-hex>.…`); a changed block churns one such pair + per block, so the default display groups them into a count. -/ +private def isSyntheticMuts (s : String) : Bool := + match s.splitOn "." with + | "Ix" :: h :: _ => + h.length == 64 && h.all fun c => c.isDigit || ('a' ≤ c && c ≤ 'f') + | _ => false + +private def printStats (path : String) (s : Ixon.EnvStats) : IO Unit := + IO.println + s!"[diff] {path}: {s.consts} consts, {s.named} named, {s.blobs} blobs, {s.comms} comms" + +/-- Print up to `cap` addresses (all when `verbose`), one per line. -/ +private def printAddrList + (linePrefix : String) (addrs : Array Address) (verbose : Bool) : + IO Unit := do + let cap := if verbose then addrs.size else min addrs.size 10 + for a in addrs[0:cap] do + IO.println s!"{linePrefix}{shortAddr verbose a}" + if addrs.size > cap then + IO.println s!"{linePrefix}… and {addrs.size - cap} more" + +private def brackets (labels : Array String) : String := + "[" ++ ", ".intercalate labels.toList ++ "]" + +private def printNamedSection + (d : Ixon.EnvDiff) (wantMeta verbose : Bool) : IO Unit := do + let keep (s : String) : Bool := verbose || !isSyntheticMuts s + let added := d.namedAdded.filter (keep ·.1) + let removed := d.namedRemoved.filter (keep ·.1) + let synAdded := d.namedAdded.size - added.size + let synRemoved := d.namedRemoved.size - removed.size + let metaCount := + if wantMeta then s!", {d.namedMetaOnly.size} metadata-only" else "" + IO.println + s!"named: {d.namedAdded.size} added, {d.namedRemoved.size} removed, {d.namedChanged.size} changed{metaCount}" + if synAdded + synRemoved > 0 then + IO.println + s!" (synthetic mutual-block names: {synAdded} added, {synRemoved} removed — --verbose lists)" + -- Column width over everything we are about to print. + let mut wMax := 0 + for (n, _) in added do wMax := max wMax n.length + for (n, _) in removed do wMax := max wMax n.length + for c in d.namedChanged do wMax := max wMax c.name.length + for (n, _) in d.namedMetaOnly do wMax := max wMax n.length + let w := min wMax 40 + for (n, addr) in added do + IO.println s!" + {pad n w} {shortAddr verbose addr}" + for (n, addr) in removed do + IO.println s!" - {pad n w} {shortAddr verbose addr}" + for c in d.namedChanged do + let kind := + if c.oldKind == c.newKind then c.oldKind + else s!"{c.oldKind}→{c.newKind}" + IO.println + s!" ~ {pad c.name w} {pad kind 9} {shortAddr verbose c.oldAddr} → {shortAddr verbose c.newAddr} {brackets c.fields}" + if wantMeta && !c.metaFields.isEmpty then + IO.println s!" meta: {brackets c.metaFields}" + for (n, labels) in d.namedMetaOnly do + IO.println s!" m {pad n w} {brackets labels}" + if d.namedChanged.any (·.fields.contains "encoding") then + IO.println + " (encoding = representation changed; no semantic field difference detected)" + +def runDiffCmd (p : Cli.Parsed) : IO UInt32 := do + let some oldArg := p.positionalArg? "old" + | p.printError "error: must specify "; return 2 + let some newArg := p.positionalArg? "new" + | p.printError "error: must specify "; return 2 + let oldPath := oldArg.as! String + let newPath := newArg.as! String + let wantMeta := p.hasFlag "meta" + if wantMeta && p.hasFlag "anon" then + IO.eprintln "error: --anon and --meta are mutually exclusive" + return 2 + let verbose := p.hasFlag "verbose" + let readBin (path : String) : IO (Option ByteArray) := + try pure (some (← IO.FS.readBinFile path)) + catch e => IO.eprintln s!"error: {path}: {e}"; pure none + let some oldBytes ← readBin oldPath | return 2 + let some newBytes ← readBin newPath | return 2 + if oldBytes == newBytes then + IO.println "identical" + return 0 + let d ← match Ixon.rsDiffEnvs oldBytes newBytes wantMeta with + | .error e => IO.eprintln s!"error: {e}"; return 2 + | .ok d => pure d + printStats oldPath d.statsA + printStats newPath d.statsB + if d.isEmpty then + if wantMeta then + IO.println "files differ in bytes but no semantic difference found" + else + IO.println + "files differ in bytes but no anonymous-structure difference found (try --meta)" + return 0 + if let some (oldMain, newMain) := d.mainChanged then + let fmt : Option Address → String + | none => "∅" + | some a => shortAddr verbose a + IO.println s!"main: {fmt oldMain} → {fmt newMain}" + unless d.assumptionsAdded.isEmpty && d.assumptionsRemoved.isEmpty do + IO.println + s!"assumptions: +{d.assumptionsAdded.size} −{d.assumptionsRemoved.size}" + printAddrList " + " d.assumptionsAdded verbose + printAddrList " - " d.assumptionsRemoved verbose + unless d.namedAdded.isEmpty && d.namedRemoved.isEmpty + && d.namedChanged.isEmpty && d.namedMetaOnly.isEmpty do + printNamedSection d wantMeta verbose + unless d.commsAdded.isEmpty && d.commsRemoved.isEmpty + && d.commsChanged.isEmpty do + IO.println + s!"comms: +{d.commsAdded.size} −{d.commsRemoved.size} ~{d.commsChanged.size}" + printAddrList " + " d.commsAdded verbose + printAddrList " - " d.commsRemoved verbose + printAddrList " ~ " d.commsChanged verbose + unless d.constsOnlyA.isEmpty && d.constsOnlyB.isEmpty do + let note := + if verbose then "" else " (mutual blocks/projections; --verbose lists)" + IO.println + s!"consts: {d.constsOnlyA.size} only in {oldPath}, {d.constsOnlyB.size} only in {newPath}{note}" + if verbose then + printAddrList " - " d.constsOnlyA verbose + printAddrList " + " d.constsOnlyB verbose + unless d.blobsOnlyA.isEmpty && d.blobsOnlyB.isEmpty do + IO.println + s!"blobs: {d.blobsOnlyA.size} only in {oldPath}, {d.blobsOnlyB.size} only in {newPath}" + if verbose then + printAddrList " - " d.blobsOnlyA verbose + printAddrList " + " d.blobsOnlyB verbose + unless d.hintsChanged.isEmpty do + IO.println s!"hints changed: {d.hintsChanged.size}" + let cap := if verbose then d.hintsChanged.size else min d.hintsChanged.size 10 + for (a, oldH, newH) in d.hintsChanged[0:cap] do + IO.println s!" {shortAddr verbose a} {oldH} → {newH}" + if d.hintsChanged.size > cap then + IO.println s!" … and {d.hintsChanged.size - cap} more" + IO.println s!"[diff] {oldPath} ≠ {newPath}" + return 1 + +end Ix.Cli.DiffCmd + +open Ix.Cli.DiffCmd in +def diffCmd : Cli.Cmd := `[Cli| + diff VIA runDiffCmd; + "Print a structured diff of two serialized Ixon environments (`.ixe`). By default only anonymous structure is compared: constants by content address (joined through names, with per-field change classification), consts/blobs sets, comms, main/assumptions, and reducibility hints. Exit codes: 0 = no difference, 1 = differences found, 2 = error." + + FLAGS: + anon; "Compare only anonymous structure (the default; accepted for explicitness)." + «meta»; "Additionally compare named metadata (binder names, originals, kv-maps)." + verbose; "Print full addresses, uncapped lists, and synthetic mutual-block names." + + ARGS: + old : String; "Path to the first (old) serialized env (`.ixe`)." + new : String; "Path to the second (new) serialized env (`.ixe`)." +] + +end diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index 610c3ca57..e1a5edb48 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -2272,6 +2272,91 @@ opaque rsDeEnvAnonFFI : @& ByteArray → Except String RawEnv def rsDeEnvAnon (bytes : ByteArray) : Except String Env := return (← rsDeEnvAnonFFI bytes).toEnv +/-! ### Env diff (`rs_diff_envs`) + +The diff is computed in Rust (`ixon::diff::diff_envs`): both inputs are +parsed with the full reader (so `Named.original` participates), the +report below is marshaled back with names pre-rendered (`Name::pretty`) +and pre-sorted, addresses raw. Field order in these structures matters: +the Rust builders address constructor slots positionally (see +`crates/ffi/src/lean_ixon/diff.rs` and the `LeanIxonEnvDiff` layout in +`crates/ffi/src/lean.rs`). -/ + +/-- Per-env entity counts for the diff header. -/ +structure EnvStats where + consts : UInt64 + named : UInt64 + blobs : UInt64 + comms : UInt64 + deriving Inhabited, BEq + +/-- One name present in both envs whose constant address changed. + `fields` is never empty: `"kind"` marks a variant change and + `"encoding"` an address change with no detected semantic field + difference (table reorder / sharing-decision churn). `metaFields` + is only populated in meta mode. -/ +structure NamedDiff where + name : String + oldAddr : Address + newAddr : Address + oldKind : String + newKind : String + fields : Array String + metaFields : Array String + deriving Inhabited, BEq + +/-- Report produced by `rsDiffEnvs`. Set-difference lists are complete + (display layers cap as needed); name-keyed arrays are sorted by + pretty name, address arrays ascending. -/ +structure EnvDiff where + /-- `none` = unchanged, otherwise `(first env's main, second's)`. -/ + mainChanged : Option (Option Address × Option Address) + assumptionsAdded : Array Address + assumptionsRemoved : Array Address + namedAdded : Array (String × Address) + namedRemoved : Array (String × Address) + namedChanged : Array NamedDiff + /-- Same constant address, different metadata (meta mode only). -/ + namedMetaOnly : Array (String × Array String) + commsAdded : Array Address + commsRemoved : Array Address + commsChanged : Array Address + constsOnlyA : Array Address + constsOnlyB : Array Address + blobsOnlyA : Array Address + blobsOnlyB : Array Address + /-- Hint deltas for constants present in BOTH envs, rendered as + `"opaque" | "abbrev" | "regular(N)" | "none"`. -/ + hintsChanged : Array (Address × String × String) + statsA : EnvStats + statsB : EnvStats + deriving Inhabited, BEq + +/-- True when no difference was found (ignores `statsA`/`statsB`, + which are always populated). -/ +def EnvDiff.isEmpty (d : EnvDiff) : Bool := + d.mainChanged.isNone + && d.assumptionsAdded.isEmpty && d.assumptionsRemoved.isEmpty + && d.namedAdded.isEmpty && d.namedRemoved.isEmpty + && d.namedChanged.isEmpty && d.namedMetaOnly.isEmpty + && d.commsAdded.isEmpty && d.commsRemoved.isEmpty && d.commsChanged.isEmpty + && d.constsOnlyA.isEmpty && d.constsOnlyB.isEmpty + && d.blobsOnlyA.isEmpty && d.blobsOnlyB.isEmpty + && d.hintsChanged.isEmpty + +@[extern "rs_diff_envs"] +opaque rsDiffEnvsFFI : @& ByteArray → @& ByteArray → Bool → Except String EnvDiff + +/-- Diff two serialized envs in Rust. `compareMeta := false` (the + default) compares only anonymous structure — name→addr changes with + per-field classification, consts/blobs sets, comms, + `main`/`assumptions`, and reducibility hints; `compareMeta := true` + additionally compares `Named` metadata content (`namedMetaOnly` + + `NamedDiff.metaFields`). -/ +def rsDiffEnvs (a b : ByteArray) (compareMeta : Bool := false) : + Except String EnvDiff := + rsDiffEnvsFFI a b compareMeta + /-! ## Canonical merkle root over consts -/ @[extern "rs_env_merkle_root"] diff --git a/Main.lean b/Main.lean index 7bdfd5b88..1ed9e0c07 100644 --- a/Main.lean +++ b/Main.lean @@ -6,6 +6,7 @@ import Ix.Cli.CodegenCmd import Ix.Cli.CheckRsCmd import Ix.Cli.ClaimCmd import Ix.Cli.CompileCmd +import Ix.Cli.DiffCmd import Ix.Cli.IngressCmd import Ix.Cli.ProfileCmd import Ix.Cli.ProveCmd @@ -31,6 +32,7 @@ def ixCmd : Cli.Cmd := `[Cli| checkCmd; checkRsCmd; claimCmd; + diffCmd; treeCmd; profileCmd; proveCmd; diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index 63c988cd1..a62d687e7 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -266,6 +266,103 @@ def rawEnvTests : TestSeq := test "RawEnv empty" (rawEnvEq (roundtripRawEnv empty) empty) ++ test "RawEnv with data" (rawEnvEq (roundtripRawEnv withData) withData) +/-! ## Env diff FFI (`rs_diff_envs`) + +The classification logic is covered by Rust unit tests +(`crates/ixon/src/diff.rs`); these tests pin the FFI marshaling — a +constructor-slot swap would surface as wrong-category contents. -/ + +def envDiffTests : TestSeq := + let fooName := Ix.Name.mkStr Ix.Name.mkAnon "foo" + let barName := Ix.Name.mkStr Ix.Name.mkAnon "bar" + -- `.var` exprs only: classification resolves table indices, so a + -- `.sort 0` over an empty univs table would be a (correctly) rejected + -- malformed constant. + let mkConst (value : Expr) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value } + sharing := #[], refs := #[], univs := #[] } + -- Named metadata carries the hint; `anonHints` stays empty so the + -- writer derives §3 from the named `Def` metadata. + let mkEnv (c : Constant) (h : Lean.ReducibilityHints) : Env := Id.run do + let addr := Address.blake3 (serConstant c) + let mut env : Env := {} + env := env.storeConst addr c + env := { env with names := RawEnv.addNameComponents env.names fooName } + env := env.registerName fooName + { addr, constMeta := .defn fooName.getHash #[] h #[] #[] {} 0 0 } + return env + let constA := mkConst (.var 0) + let constB := mkConst (.var 1) + let addrA := Address.blake3 (serConstant constA) + let addrB := Address.blake3 (serConstant constB) + let envBase := mkEnv constA (.regular 5) + let envValueChanged := mkEnv constB (.regular 5) + let envHintChanged := mkEnv constA (.regular 6) + let envExtra := Id.run do + let mut env := envBase + env := env.storeConst addrB constB + env := { env with names := RawEnv.addNameComponents env.names barName } + env := env.registerName barName { addr := addrB, constMeta := .empty } + return env + let runDiff (a b : Env) (compareMeta : Bool := false) : + Option Ixon.EnvDiff := + (Ixon.rsDiffEnvs (serEnv a) (serEnv b) compareMeta).toOption + test "EnvDiff: self-diff empty (anon)" + ((runDiff envBase envBase).any (·.isEmpty)) ++ + test "EnvDiff: self-diff empty (meta)" + ((runDiff envBase envBase true).any (·.isEmpty)) ++ + test "EnvDiff: stats populated" + ((runDiff envBase envBase).any fun d => + d.statsA.consts == 1 && d.statsA.named == 1 && d.statsB == d.statsA) ++ + test "EnvDiff: added name" + ((runDiff envBase envExtra).any fun d => + d.namedAdded == #[("bar", addrB)] && d.constsOnlyB == #[addrB] + && d.namedRemoved.isEmpty && d.namedChanged.isEmpty) ++ + test "EnvDiff: removed name" + ((runDiff envExtra envBase).any fun d => + d.namedRemoved == #[("bar", addrB)] && d.constsOnlyA == #[addrB]) ++ + test "EnvDiff: value change classified" + ((runDiff envBase envValueChanged).any fun d => + d.namedChanged.size == 1 && + (d.namedChanged[0]!).name == "foo" && + (d.namedChanged[0]!).oldAddr == addrA && + (d.namedChanged[0]!).newAddr == addrB && + (d.namedChanged[0]!).oldKind == "defn" && + (d.namedChanged[0]!).newKind == "defn" && + (d.namedChanged[0]!).fields == #["value"] && + (d.namedChanged[0]!).metaFields.isEmpty) ++ + -- Hints derive into anon §3, so a hint tweak is visible in the + -- default anon mode; the metadata carrying it only shows under meta. + test "EnvDiff: hint change visible in anon mode" + ((runDiff envBase envHintChanged).any fun d => + d.hintsChanged == #[(addrA, "regular(5)", "regular(6)")] + && d.namedMetaOnly.isEmpty && d.namedChanged.isEmpty) ++ + test "EnvDiff: hint change flags metadata under meta mode" + ((runDiff envBase envHintChanged true).any fun d => + d.hintsChanged.size == 1 + && d.namedMetaOnly == #[("foo", #["meta.info"])]) ++ + test "EnvDiff: main change" + ((runDiff { envBase with main := some addrA } envBase).any fun d => + d.mainChanged == some (some addrA, none)) ++ + test "EnvDiff: assumptions delta" + (let p := Address.blake3 (ByteArray.mk #[1]) + let q := Address.blake3 (ByteArray.mk #[2]) + let r := Address.blake3 (ByteArray.mk #[3]) + let a := { envBase with + assumptions := ({} : Std.HashSet Address).insert p |>.insert q } + let b := { envBase with + assumptions := ({} : Std.HashSet Address).insert q |>.insert r } + (runDiff a b).any fun d => + d.assumptionsAdded == #[r] && d.assumptionsRemoved == #[p]) + +/-- Self-diff over the pure-Lean writer's bytes must be empty in both + modes (also exercises report marshaling on arbitrary inputs). -/ +def selfDiffEmpty (compareMeta : Bool) (env : RawEnv) : Bool := + match Ixon.rsDiffEnvs (serEnv env.toEnv) (serEnv env.toEnv) compareMeta with + | .ok d => d.isEmpty + | .error _ => false + /-! ## Test Suite -/ def suite : List TestSeq := [ @@ -307,6 +404,12 @@ def suite : List TestSeq := [ checkIO "Ixon.Named roundtrip" (∀ x : Named, roundtripIxonNamed x == x), ---- RawEnv roundtrip checkIO "Ixon.RawEnv roundtrip" (∀ env : RawEnv, rawEnvEq (roundtripRawEnv env) env), + ---- Env diff + envDiffTests, + checkIO "Ixon.EnvDiff self-diff empty (anon)" + (∀ env : RawEnv, selfDiffEmpty false env), + checkIO "Ixon.EnvDiff self-diff empty (meta)" + (∀ env : RawEnv, selfDiffEmpty true env), ] end Tests.FFI.Ixon diff --git a/crates/ffi/src/lean.rs b/crates/ffi/src/lean.rs index cb920c9f9..f3fb24afa 100644 --- a/crates/ffi/src/lean.rs +++ b/crates/ffi/src/lean.rs @@ -62,6 +62,16 @@ lean_ffi::lean_inductive! { LeanIxonRawNamedLite [ { num_obj: 2, num_64: 2 } ]; LeanIxonRawEnvLazy [ { num_obj: 5 } ]; + // Env diff report (`rs_diff_envs`). Slot counts MUST match the Lean + // structures in Ix/Ixon.lean (Ixon.EnvStats / NamedDiff / EnvDiff). + // consts, named, blobs, comms counts + LeanIxonEnvStats [ { num_64: 4 } ]; + // name, oldAddr, newAddr, oldKind, newKind, fields, metaFields + LeanIxonNamedDiff [ { num_obj: 7 } ]; + // mainChanged, assumptions±, named±/changed/metaOnly, comms±/changed, + // constsOnly×2, blobsOnly×2, hintsChanged, statsA, statsB + LeanIxonEnvDiff [ { num_obj: 17 } ]; + // --- Ixon multi-variant inductives --- LeanIxonUniv [ diff --git a/crates/ffi/src/lean_ixon.rs b/crates/ffi/src/lean_ixon.rs index 51a89fd04..742167dda 100644 --- a/crates/ffi/src/lean_ixon.rs +++ b/crates/ffi/src/lean_ixon.rs @@ -6,6 +6,7 @@ #[cfg(feature = "test-ffi")] pub mod compare; pub mod constant; +pub mod diff; pub mod enums; pub mod env; pub mod expr; diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs new file mode 100644 index 000000000..62b2c7e3c --- /dev/null +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -0,0 +1,165 @@ +//! `rs_diff_envs`: parse two serialized envs with the full reader +//! (`Env::get`, preserving `Named.original`), diff them in Rust +//! (`ixon::diff::diff_envs`), and marshal the [`EnvDiff`] report to +//! Lean (`Ixon.EnvDiff`). +//! +//! Names cross the boundary pre-rendered (`Name::pretty()`) and +//! pre-sorted; addresses cross raw so Lean owns display formatting. + +use ix_common::address::Address; +use ixon::diff::{EnvDiff, EnvStats, NamedChange, diff_envs}; +use ixon::env::Env as IxonEnv; +use lean_ffi::object::{ + LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, + LeanOwned, LeanProd, LeanString, +}; + +use crate::lean::{ + LeanIxAddress, LeanIxonEnvDiff, LeanIxonEnvStats, LeanIxonNamedDiff, +}; + +fn addr_array(v: &[Address]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, a) in v.iter().enumerate() { + arr.set(i, LeanIxAddress::build(a)); + } + arr +} + +fn string_array(v: &[String]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, s) in v.iter().enumerate() { + arr.set(i, LeanString::new(s)); + } + arr +} + +/// `Array (String × Address)` +fn name_addr_array(v: &[(String, Address)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (s, a)) in v.iter().enumerate() { + arr.set(i, LeanProd::new(LeanString::new(s), LeanIxAddress::build(a))); + } + arr +} + +/// `Array (String × Array String)` +fn name_labels_array(v: &[(String, Vec)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (s, labels)) in v.iter().enumerate() { + arr.set(i, LeanProd::new(LeanString::new(s), string_array(labels))); + } + arr +} + +/// `Array (Address × String × String)` (right-nested prod) +fn hint_array(v: &[(Address, String, String)]) -> LeanArray { + let arr = LeanArray::alloc(v.len()); + for (i, (a, old, new)) in v.iter().enumerate() { + arr.set( + i, + LeanProd::new( + LeanIxAddress::build(a), + LeanProd::new(LeanString::new(old), LeanString::new(new)), + ), + ); + } + arr +} + +fn opt_addr(o: &Option
) -> LeanOwned { + match o { + None => LeanOption::none().into(), + Some(a) => LeanOption::some(LeanIxAddress::build(a)).into(), + } +} + +impl LeanIxonEnvStats { + /// Build `Ixon.EnvStats { consts, named, blobs, comms }`. + fn build(s: &EnvStats) -> Self { + let ctor = LeanIxonEnvStats::alloc(0); + ctor.set_num_64(0, s.consts as u64); + ctor.set_num_64(1, s.named as u64); + ctor.set_num_64(2, s.blobs as u64); + ctor.set_num_64(3, s.comms as u64); + ctor + } +} + +impl LeanIxonNamedDiff { + /// Build `Ixon.NamedDiff { name, oldAddr, newAddr, oldKind, newKind, + /// fields, metaFields }`. + fn build(c: &NamedChange) -> Self { + let ctor = LeanIxonNamedDiff::alloc(0); + ctor.set_obj(0, LeanString::new(&c.name)); + ctor.set_obj(1, LeanIxAddress::build(&c.old_addr)); + ctor.set_obj(2, LeanIxAddress::build(&c.new_addr)); + ctor.set_obj(3, LeanString::new(c.old_kind)); + ctor.set_obj(4, LeanString::new(c.new_kind)); + ctor.set_obj(5, string_array(&c.fields)); + ctor.set_obj(6, string_array(&c.meta_fields)); + ctor + } +} + +impl LeanIxonEnvDiff { + /// Build `Ixon.EnvDiff` — field order MUST match the Lean structure. + fn build(d: &EnvDiff) -> Self { + let main_changed: LeanOwned = match &d.main_changed { + None => LeanOption::none().into(), + Some((a, b)) => { + LeanOption::some(LeanProd::new(opt_addr(a), opt_addr(b))).into() + }, + }; + let named_changed = LeanArray::alloc(d.named_changed.len()); + for (i, c) in d.named_changed.iter().enumerate() { + named_changed.set(i, LeanIxonNamedDiff::build(c)); + } + let ctor = LeanIxonEnvDiff::alloc(0); + ctor.set_obj(0, main_changed); + ctor.set_obj(1, addr_array(&d.assumptions_added)); + ctor.set_obj(2, addr_array(&d.assumptions_removed)); + ctor.set_obj(3, name_addr_array(&d.named_added)); + ctor.set_obj(4, name_addr_array(&d.named_removed)); + ctor.set_obj(5, named_changed); + ctor.set_obj(6, name_labels_array(&d.named_meta_only)); + ctor.set_obj(7, addr_array(&d.comms_added)); + ctor.set_obj(8, addr_array(&d.comms_removed)); + ctor.set_obj(9, addr_array(&d.comms_changed)); + ctor.set_obj(10, addr_array(&d.consts_only_a)); + ctor.set_obj(11, addr_array(&d.consts_only_b)); + ctor.set_obj(12, addr_array(&d.blobs_only_a)); + ctor.set_obj(13, addr_array(&d.blobs_only_b)); + ctor.set_obj(14, hint_array(&d.hints_changed)); + ctor.set_obj(15, LeanIxonEnvStats::build(&d.stats_a)); + ctor.set_obj(16, LeanIxonEnvStats::build(&d.stats_b)); + ctor + } +} + +/// FFI: diff two serialized envs. +/// `(a b : ByteArray) → (meta : Bool) → Except String Ixon.EnvDiff`. +/// Pure; both buffers are parsed with the full reader (`Env::get`). +#[unsafe(no_mangle)] +pub extern "C" fn rs_diff_envs( + a: LeanByteArray>, + b: LeanByteArray>, + meta: LeanBool>, +) -> LeanExcept { + let parse = |bytes: &[u8], which: &str| -> Result { + let mut cursor: &[u8] = bytes; + IxonEnv::get(&mut cursor).map_err(|e| format!("{which} input: {e}")) + }; + let env_a = match parse(a.as_bytes(), "first") { + Ok(env) => env, + Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + }; + let env_b = match parse(b.as_bytes(), "second") { + Ok(env) => env, + Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + }; + match diff_envs(&env_a, &env_b, meta.to_bool()) { + Ok(d) => LeanExcept::ok(LeanIxonEnvDiff::build(&d)), + Err(e) => LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + } +} diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs new file mode 100644 index 000000000..e7045d883 --- /dev/null +++ b/crates/ixon/src/diff.rs @@ -0,0 +1,1375 @@ +//! Structured diff of two environments. +//! +//! [`diff_envs`] compares two [`Env`]s and produces an [`EnvDiff`] report. +//! Content-addressing frames the comparison: a name "changed" iff its +//! `Named.addr` differs, and constant value-equality coincides with +//! address-equality. Because addresses hash the *representation* +//! (sharing/refs/univs tables included), a changed address may carry no +//! semantic field difference — table reordering or different sharing +//! decisions — which the classifier reports honestly as `"encoding"`. +//! +//! Two modes: +//! - anon (`meta = false`, the default): only anonymous structure is +//! compared — name→addr changes (with per-field classification of the +//! two constants), consts/blobs set differences, comms, `main`, +//! `assumptions`, and reducibility hints (which live in the anon §3 +//! section and drive kernel unfolding). Names serve as join/display +//! keys only. +//! - meta (`meta = true`): additionally compares `Named.meta` / +//! `Named.original` content, populating [`EnvDiff::named_meta_only`] +//! and [`NamedChange::meta_fields`]. +//! +//! Expression comparison ([`exprs_equal`]) is a lockstep structural walk +//! that resolves table indices through each side's own `Constant` tables: +//! identical expr bytes over different tables compare *different*, and +//! different indices resolving to the same addresses with the same +//! structure compare *equal*. +//! +//! Out of scope: `Env.names` (hash-consed name components — derived data; +//! the named join subsumes user-visible changes) and explanations for +//! orphan consts (mutual blocks and aux originals appear in the consts +//! set difference as raw truth). Synthetic Muts names (`Ix.` +//! prefixed) churn as one removed+added pair per changed block; display +//! layers may group them. + +use rustc_hash::FxHashSet; + +use ix_common::address::Address; +use ix_common::env::{Name, ReducibilityHints}; + +use super::constant::{ + Constant, ConstantInfo, Constructor, Definition, Inductive, MutConst, + Recursor, +}; +use super::env::{Env, Named}; +use super::expr::Expr; +use super::univ::Univ; + +/// Per-env entity counts, reported for both inputs so display layers can +/// print a header without re-walking the envs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct EnvStats { + pub consts: usize, + pub named: usize, + pub blobs: usize, + pub comms: usize, +} + +/// One name present in both envs whose constant address changed. +#[derive(Clone, Debug)] +pub struct NamedChange { + /// `Name::pretty()` of the joined name. + pub name: String, + pub old_addr: Address, + pub new_addr: Address, + /// [`kind_label`] of each side's `ConstantInfo` variant. + pub old_kind: &'static str, + pub new_kind: &'static str, + /// Changed-field labels, e.g. `"value"`, `"block.ctors[0].type"`. + /// Never empty: `"kind"` when the variants differ, `"encoding"` when + /// no semantic field difference explains the address change. + pub fields: Vec, + /// Metadata component labels when the metadata ALSO changed. Only + /// populated in meta mode; may be empty. + pub meta_fields: Vec, +} + +/// Report produced by [`diff_envs`]. All name-keyed vectors are sorted +/// by (pretty name, name hash); address vectors are sorted ascending. +/// Set-difference lists are complete — display layers cap as needed. +#[derive(Clone, Debug, Default)] +pub struct EnvDiff { + /// `None` = unchanged, otherwise `(a.main, b.main)`. + pub main_changed: Option<(Option
, Option
)>, + pub assumptions_added: Vec
, + pub assumptions_removed: Vec
, + /// Names only in the second env, with their constant address. + pub named_added: Vec<(String, Address)>, + /// Names only in the first env, with their constant address. + pub named_removed: Vec<(String, Address)>, + pub named_changed: Vec, + /// Same constant address, different metadata (meta mode only). + pub named_meta_only: Vec<(String, Vec)>, + pub comms_added: Vec
, + pub comms_removed: Vec
, + /// Same commitment address, different `Comm` payload (possible + /// because the comms section is not hash-verified on load). + pub comms_changed: Vec
, + pub consts_only_a: Vec
, + pub consts_only_b: Vec
, + pub blobs_only_a: Vec
, + pub blobs_only_b: Vec
, + /// Hint deltas for constants present in BOTH envs (hint deltas of + /// added/removed constants are implied by the other sections). + /// Rendered as `"opaque" | "abbrev" | "regular(N)" | "none"`. + pub hints_changed: Vec<(Address, String, String)>, + pub stats_a: EnvStats, + pub stats_b: EnvStats, +} + +impl EnvDiff { + /// True when no difference was found. Ignores `stats_*` (always + /// populated). + pub fn is_empty(&self) -> bool { + self.main_changed.is_none() + && self.assumptions_added.is_empty() + && self.assumptions_removed.is_empty() + && self.named_added.is_empty() + && self.named_removed.is_empty() + && self.named_changed.is_empty() + && self.named_meta_only.is_empty() + && self.comms_added.is_empty() + && self.comms_removed.is_empty() + && self.comms_changed.is_empty() + && self.consts_only_a.is_empty() + && self.consts_only_b.is_empty() + && self.blobs_only_a.is_empty() + && self.blobs_only_b.is_empty() + && self.hints_changed.is_empty() + } +} + +/// Short label for a `ConstantInfo` variant. +pub fn kind_label(info: &ConstantInfo) -> &'static str { + match info { + ConstantInfo::Defn(_) => "defn", + ConstantInfo::Recr(_) => "recr", + ConstantInfo::Axio(_) => "axio", + ConstantInfo::Quot(_) => "quot", + ConstantInfo::CPrj(_) => "cprj", + ConstantInfo::RPrj(_) => "rprj", + ConstantInfo::IPrj(_) => "iprj", + ConstantInfo::DPrj(_) => "dprj", + ConstantInfo::Muts(_) => "muts", + } +} + +// ============================================================================ +// Expression comparison +// ============================================================================ + +/// Follow a `Share` chain through `c.sharing` until a non-`Share` node. +/// Hop budget guards corrupt cycles (honest constants only reference +/// strictly earlier sharing entries, so chains are finite). +fn resolve_share<'c>( + mut e: &'c Expr, + c: &'c Constant, + side: &str, +) -> Result<&'c Expr, String> { + let mut hops: usize = 0; + while let Expr::Share(i) = e { + hops += 1; + if hops > c.sharing.len() + 1 { + return Err(format!("diff: Share cycle detected on the {side} side")); + } + let idx = usize::try_from(*i).map_err(|_e| { + format!("diff: Share index {i} overflows usize on the {side} side") + })?; + e = c.sharing.get(idx).map(|a| a.as_ref()).ok_or_else(|| { + format!( + "diff: Share index {i} out of bounds ({} sharing entries) on the {side} side", + c.sharing.len() + ) + })?; + } + Ok(e) +} + +fn univ_at<'c>( + c: &'c Constant, + idx: u64, + side: &str, +) -> Result<&'c Univ, String> { + usize::try_from(idx) + .ok() + .and_then(|i| c.univs.get(i)) + .map(|u| u.as_ref()) + .ok_or_else(|| { + format!( + "diff: univ index {idx} out of bounds ({} entries) on the {side} side", + c.univs.len() + ) + }) +} + +fn ref_at<'c>( + c: &'c Constant, + idx: u64, + side: &str, +) -> Result<&'c Address, String> { + usize::try_from(idx).ok().and_then(|i| c.refs.get(i)).ok_or_else(|| { + format!( + "diff: ref index {idx} out of bounds ({} entries) on the {side} side", + c.refs.len() + ) + }) +} + +/// Lockstep expression comparator over one pair of table contexts. +/// +/// The pointer-pair memo persists across `eq` calls for the same +/// constant pair (tables are constant-level, so shared subterms recur +/// across type/value/rules). Entries record "heads matched, children +/// enqueued" — an unequal result returns early and leaves enqueued +/// children unverified, so [`Self::eq`] clears the memo on `false`. +struct ExprCmp<'a> { + ca: &'a Constant, + cb: &'a Constant, + memo: FxHashSet<(usize, usize)>, +} + +impl<'a> ExprCmp<'a> { + fn new(ca: &'a Constant, cb: &'a Constant) -> Self { + ExprCmp { ca, cb, memo: FxHashSet::default() } + } + + fn eq(&mut self, a: &'a Expr, b: &'a Expr) -> Result { + let r = self.eq_inner(a, b)?; + if !r { + self.memo.clear(); + } + Ok(r) + } + + fn eq_inner(&mut self, a: &'a Expr, b: &'a Expr) -> Result { + // Iterative walk: kernel terms nest thousands deep, never recurse. + let mut stack: Vec<(&'a Expr, &'a Expr)> = vec![(a, b)]; + while let Some((ea, eb)) = stack.pop() { + let ea = resolve_share(ea, self.ca, "left")?; + let eb = resolve_share(eb, self.cb, "right")?; + let key = + (std::ptr::from_ref(ea) as usize, std::ptr::from_ref(eb) as usize); + if !self.memo.insert(key) { + continue; + } + match (ea, eb) { + (Expr::Var(x), Expr::Var(y)) => { + if x != y { + return Ok(false); + } + }, + (Expr::Sort(i), Expr::Sort(j)) => { + if univ_at(self.ca, *i, "left")? != univ_at(self.cb, *j, "right")? { + return Ok(false); + } + }, + (Expr::Ref(i, us), Expr::Ref(j, vs)) => { + if ref_at(self.ca, *i, "left")? != ref_at(self.cb, *j, "right")? + || !self.univ_lists_equal(us, vs)? + { + return Ok(false); + } + }, + (Expr::Rec(i, us), Expr::Rec(j, vs)) => { + // Intra-block member index: positional, compare raw. + if i != j || !self.univ_lists_equal(us, vs)? { + return Ok(false); + } + }, + (Expr::Str(i), Expr::Str(j)) | (Expr::Nat(i), Expr::Nat(j)) => { + // Blob content behind equal addresses is identical by + // content-addressing (blob sections are hash-verified on load). + if ref_at(self.ca, *i, "left")? != ref_at(self.cb, *j, "right")? { + return Ok(false); + } + }, + (Expr::Prj(ti, fi, va), Expr::Prj(tj, fj, vb)) => { + if fi != fj + || ref_at(self.ca, *ti, "left")? != ref_at(self.cb, *tj, "right")? + { + return Ok(false); + } + stack.push((va, vb)); + }, + (Expr::App(f1, x1), Expr::App(f2, x2)) + | (Expr::Lam(f1, x1), Expr::Lam(f2, x2)) + | (Expr::All(f1, x1), Expr::All(f2, x2)) => { + stack.push((f1, f2)); + stack.push((x1, x2)); + }, + (Expr::Let(n1, t1, v1, b1), Expr::Let(n2, t2, v2, b2)) => { + if n1 != n2 { + return Ok(false); + } + stack.push((t1, t2)); + stack.push((v1, v2)); + stack.push((b1, b2)); + }, + _ => return Ok(false), + } + } + Ok(true) + } + + fn univ_lists_equal(&self, us: &[u64], vs: &[u64]) -> Result { + if us.len() != vs.len() { + return Ok(false); + } + for (i, j) in us.iter().zip(vs.iter()) { + // Univ trees are self-contained (Univ::Var is a universe-parameter + // de Bruijn index, not a table index), so deep PartialEq is right. + if univ_at(self.ca, *i, "left")? != univ_at(self.cb, *j, "right")? { + return Ok(false); + } + } + Ok(true) + } +} + +/// Structural equality of two expressions, resolving `Share`/`Sort`/ +/// `Ref`/`Rec`-univ/`Str`/`Nat`/`Prj` indices through each side's own +/// `Constant` tables. Errors on out-of-bounds indices or `Share` cycles +/// (corrupt constants). +pub fn exprs_equal( + a: &Expr, + ca: &Constant, + b: &Expr, + cb: &Constant, +) -> Result { + ExprCmp::new(ca, cb).eq(a, b) +} + +// ============================================================================ +// Per-kind field classification +// ============================================================================ + +fn classify_defn<'x>( + a: &'x Definition, + b: &'x Definition, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.kind != b.kind { + out.push(format!("{prefix}def-kind")); + } + if a.safety != b.safety { + out.push(format!("{prefix}safety")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if !cmp.eq(&a.value, &b.value)? { + out.push(format!("{prefix}value")); + } + Ok(()) +} + +fn classify_recr<'x>( + a: &'x Recursor, + b: &'x Recursor, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.k != b.k { + out.push(format!("{prefix}k")); + } + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.indices != b.indices { + out.push(format!("{prefix}indices")); + } + if a.motives != b.motives { + out.push(format!("{prefix}motives")); + } + if a.minors != b.minors { + out.push(format!("{prefix}minors")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if a.rules.len() != b.rules.len() { + out.push(format!("{prefix}rules.len")); + } + for (i, (ra, rb)) in a.rules.iter().zip(b.rules.iter()).enumerate() { + if ra.fields != rb.fields { + out.push(format!("{prefix}rules[{i}].fields")); + } + if !cmp.eq(&ra.rhs, &rb.rhs)? { + out.push(format!("{prefix}rules[{i}].rhs")); + } + } + Ok(()) +} + +fn classify_ctor<'x>( + a: &'x Constructor, + b: &'x Constructor, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.cidx != b.cidx { + out.push(format!("{prefix}cidx")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.fields != b.fields { + out.push(format!("{prefix}fields")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + Ok(()) +} + +fn classify_indc<'x>( + a: &'x Inductive, + b: &'x Inductive, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if a.is_unsafe != b.is_unsafe { + out.push(format!("{prefix}unsafe")); + } + if a.lvls != b.lvls { + out.push(format!("{prefix}lvls")); + } + if a.params != b.params { + out.push(format!("{prefix}params")); + } + if a.indices != b.indices { + out.push(format!("{prefix}indices")); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push(format!("{prefix}type")); + } + if a.ctors.len() != b.ctors.len() { + out.push(format!("{prefix}ctors.len")); + } + for (j, (ca, cb)) in a.ctors.iter().zip(b.ctors.iter()).enumerate() { + classify_ctor(ca, cb, cmp, &format!("{prefix}ctors[{j}]."), out)?; + } + Ok(()) +} + +fn classify_mut_member<'x>( + a: &'x MutConst, + b: &'x MutConst, + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + match (a, b) { + (MutConst::Defn(x), MutConst::Defn(y)) => { + classify_defn(x, y, cmp, prefix, out) + }, + (MutConst::Indc(x), MutConst::Indc(y)) => { + classify_indc(x, y, cmp, prefix, out) + }, + (MutConst::Recr(x), MutConst::Recr(y)) => { + classify_recr(x, y, cmp, prefix, out) + }, + _ => { + out.push(format!("{prefix}kind")); + Ok(()) + }, + } +} + +fn classify_muts<'x>( + ma: &'x [MutConst], + mb: &'x [MutConst], + cmp: &mut ExprCmp<'x>, + prefix: &str, + out: &mut Vec, +) -> Result<(), String> { + if ma.len() != mb.len() { + out.push(format!("{prefix}members.len")); + } + for (i, (a, b)) in ma.iter().zip(mb.iter()).enumerate() { + classify_mut_member(a, b, cmp, &format!("{prefix}members[{i}]."), out)?; + } + Ok(()) +} + +/// Same-kind projection pair whose `block` differs: descend one level +/// into the two Muts blocks and classify the projected member, with +/// labels prefixed `block.`. A member identical under resolved +/// comparison means the block hash moved because a *sibling* changed — +/// reported as `"block-siblings"`. Any structural surprise (block +/// missing — e.g. behind a bundle's assumption cut —, unparseable, not +/// Muts, index out of range) falls back to the honest `"block"` label. +/// No deeper recursion: blocks do not nest. +fn classify_prj( + env_a: &Env, + env_b: &Env, + idx_a: u64, + idx_b: u64, + cidx: Option<(u64, u64)>, + block_a: &Address, + block_b: &Address, + out: &mut Vec, +) -> Result<(), String> { + if idx_a != idx_b { + out.push("idx".to_string()); + } + if let Some((ca, cb)) = cidx + && ca != cb + { + out.push("cidx".to_string()); + } + if block_a == block_b { + return Ok(()); + } + // With differing member coordinates the two projections target + // different members; per-member block detail would be meaningless. + if idx_a != idx_b || cidx.is_some_and(|(x, y)| x != y) { + out.push("block".to_string()); + return Ok(()); + } + let (Some(Ok(ba)), Some(Ok(bb))) = + (env_a.try_get_const(block_a), env_b.try_get_const(block_b)) + else { + out.push("block".to_string()); + return Ok(()); + }; + let (ConstantInfo::Muts(ma), ConstantInfo::Muts(mb)) = (&ba.info, &bb.info) + else { + out.push("block".to_string()); + return Ok(()); + }; + let Some(idx) = usize::try_from(idx_a).ok() else { + out.push("block".to_string()); + return Ok(()); + }; + let (Some(mem_a), Some(mem_b)) = (ma.get(idx), mb.get(idx)) else { + out.push("block".to_string()); + return Ok(()); + }; + // Expression tables are constant-level: members resolve through the + // enclosing block constants' tables. + let mut cmp = ExprCmp::new(&ba, &bb); + let before = out.len(); + match cidx { + None => classify_mut_member(mem_a, mem_b, &mut cmp, "block.", out)?, + Some((cx, _)) => { + let (MutConst::Indc(ia), MutConst::Indc(ib)) = (mem_a, mem_b) else { + out.push("block".to_string()); + return Ok(()); + }; + let Some(ci) = usize::try_from(cx).ok() else { + out.push("block".to_string()); + return Ok(()); + }; + let (Some(ctor_a), Some(ctor_b)) = (ia.ctors.get(ci), ib.ctors.get(ci)) + else { + out.push("block".to_string()); + return Ok(()); + }; + classify_ctor(ctor_a, ctor_b, &mut cmp, "block.ctor.", out)?; + }, + } + if out.len() == before { + out.push("block-siblings".to_string()); + } + Ok(()) +} + +/// Compare two same-name constants field by field; returns changed-field +/// labels. Never returns an empty list: kind mismatches yield `"kind"`, +/// and an address change with no detected semantic difference yields +/// `"encoding"` (table reorder / sharing-decision churn). +fn classify_constants( + env_a: &Env, + ca: &Constant, + env_b: &Env, + cb: &Constant, +) -> Result, String> { + let mut out = Vec::new(); + let mut cmp = ExprCmp::new(ca, cb); + match (&ca.info, &cb.info) { + (ConstantInfo::Defn(a), ConstantInfo::Defn(b)) => { + classify_defn(a, b, &mut cmp, "", &mut out)?; + }, + (ConstantInfo::Recr(a), ConstantInfo::Recr(b)) => { + classify_recr(a, b, &mut cmp, "", &mut out)?; + }, + (ConstantInfo::Axio(a), ConstantInfo::Axio(b)) => { + if a.is_unsafe != b.is_unsafe { + out.push("unsafe".to_string()); + } + if a.lvls != b.lvls { + out.push("lvls".to_string()); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push("type".to_string()); + } + }, + (ConstantInfo::Quot(a), ConstantInfo::Quot(b)) => { + if a.kind != b.kind { + out.push("quot-kind".to_string()); + } + if a.lvls != b.lvls { + out.push("lvls".to_string()); + } + if !cmp.eq(&a.typ, &b.typ)? { + out.push("type".to_string()); + } + }, + (ConstantInfo::DPrj(a), ConstantInfo::DPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + )?; + }, + (ConstantInfo::IPrj(a), ConstantInfo::IPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + )?; + }, + (ConstantInfo::RPrj(a), ConstantInfo::RPrj(b)) => { + classify_prj( + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + )?; + }, + (ConstantInfo::CPrj(a), ConstantInfo::CPrj(b)) => { + classify_prj( + env_a, + env_b, + a.idx, + b.idx, + Some((a.cidx, b.cidx)), + &a.block, + &b.block, + &mut out, + )?; + }, + (ConstantInfo::Muts(ma), ConstantInfo::Muts(mb)) => { + classify_muts(ma, mb, &mut cmp, "", &mut out)?; + }, + _ => out.push("kind".to_string()), + } + if out.is_empty() { + out.push("encoding".to_string()); + } + Ok(out) +} + +// ============================================================================ +// Metadata comparison (meta mode) +// ============================================================================ + +/// Component labels for a same-addr metadata difference. +fn meta_component_labels(a: &Named, b: &Named) -> Vec { + let mut out = Vec::new(); + let (am, bm) = (a.meta(), b.meta()); + if am.info != bm.info { + let (ka, kb) = (am.info.kind_name(), bm.info.kind_name()); + if ka == kb { + out.push("meta.info".to_string()); + } else { + out.push(format!("meta.info({ka}→{kb})")); + } + } + if am.meta_sharing != bm.meta_sharing { + out.push("meta.sharing".to_string()); + } + if am.meta_refs != bm.meta_refs { + out.push("meta.refs".to_string()); + } + if am.meta_univs != bm.meta_univs { + out.push("meta.univs".to_string()); + } + match (a.original(), b.original()) { + (None, None) => {}, + (None, Some(_)) => out.push("original.added".to_string()), + (Some(_), None) => out.push("original.removed".to_string()), + (Some((aa, ma)), Some((ab, mb))) => { + if aa != ab { + out.push("original.addr".to_string()); + } + if ma != mb { + out.push("original.meta".to_string()); + } + }, + } + out +} + +// ============================================================================ +// Env diff +// ============================================================================ + +fn hint_label(h: Option<&ReducibilityHints>) -> String { + match h { + None => "none".to_string(), + Some(ReducibilityHints::Opaque) => "opaque".to_string(), + Some(ReducibilityHints::Abbrev) => "abbrev".to_string(), + Some(ReducibilityHints::Regular(n)) => format!("regular({n})"), + } +} + +fn env_stats(e: &Env) -> EnvStats { + EnvStats { + consts: e.consts.len(), + named: e.named.len(), + blobs: e.blobs.len(), + comms: e.comms.len(), + } +} + +fn classify_named_change( + env_a: &Env, + env_b: &Env, + name: &Name, + na: &Named, + nb: &Named, + meta: bool, +) -> Result { + let materialize = |env: &Env, addr: &Address, which: &str| { + env + .try_get_const(addr) + .ok_or_else(|| { + format!( + "diff: named '{}' addr {} not present in consts of the {which} env", + name.pretty(), + addr.hex() + ) + })? + .map_err(|e| { + format!("diff: named '{}' ({}): {e}", name.pretty(), addr.hex()) + }) + }; + let ca = materialize(env_a, &na.addr, "first")?; + let cb = materialize(env_b, &nb.addr, "second")?; + let fields = classify_constants(env_a, &ca, env_b, &cb)?; + let meta_fields = + if meta { meta_component_labels(na, nb) } else { Vec::new() }; + Ok(NamedChange { + name: name.pretty(), + old_addr: na.addr.clone(), + new_addr: nb.addr.clone(), + old_kind: kind_label(&ca.info), + new_kind: kind_label(&cb.info), + fields, + meta_fields, + }) +} + +/// Sort by (pretty name, name hash) — the hash tiebreak keeps distinct +/// names with equal pretty forms deterministic. +fn sort_by_name(v: &mut [(String, Name, T)]) { + v.sort_by(|x, y| x.0.cmp(&y.0).then_with(|| x.1.cmp(&y.1))); +} + +/// Diff two environments. `meta = false` (anon mode, the default) +/// compares only anonymous structure; `meta = true` additionally +/// compares `Named` metadata content. See the module docs. +pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { + let mut d = EnvDiff { + stats_a: env_stats(a), + stats_b: env_stats(b), + ..EnvDiff::default() + }; + + // Named join on Name. Alpha-equivalent multi-names (several names + // sharing one constant address) need no special care: each name is + // its own row. + let mut added: Vec<(String, Name, Address)> = Vec::new(); + let mut removed: Vec<(String, Name, Address)> = Vec::new(); + let mut changed: Vec<(String, Name, NamedChange)> = Vec::new(); + let mut meta_only: Vec<(String, Name, Vec)> = Vec::new(); + for entry in a.named.iter() { + let (name, na) = (entry.key(), entry.value()); + match b.named.get(name) { + None => removed.push((name.pretty(), name.clone(), na.addr.clone())), + Some(nb_ref) => { + let nb = nb_ref.value(); + if na.addr == nb.addr { + if meta { + let labels = meta_component_labels(na, nb); + if !labels.is_empty() { + meta_only.push((name.pretty(), name.clone(), labels)); + } + } + } else { + let change = classify_named_change(a, b, name, na, nb, meta)?; + changed.push((name.pretty(), name.clone(), change)); + } + }, + } + } + for entry in b.named.iter() { + let name = entry.key(); + if a.named.get(name).is_none() { + added.push((name.pretty(), name.clone(), entry.value().addr.clone())); + } + } + sort_by_name(&mut added); + sort_by_name(&mut removed); + sort_by_name(&mut changed); + sort_by_name(&mut meta_only); + d.named_added = added.into_iter().map(|(s, _, addr)| (s, addr)).collect(); + d.named_removed = removed.into_iter().map(|(s, _, addr)| (s, addr)).collect(); + d.named_changed = changed.into_iter().map(|(_, _, c)| c).collect(); + d.named_meta_only = meta_only.into_iter().map(|(s, _, l)| (s, l)).collect(); + + // Consts / blobs set differences. + for entry in a.consts.iter() { + if !b.consts.contains_key(entry.key()) { + d.consts_only_a.push(entry.key().clone()); + } + } + for entry in b.consts.iter() { + if !a.consts.contains_key(entry.key()) { + d.consts_only_b.push(entry.key().clone()); + } + } + for entry in a.blobs.iter() { + if !b.blobs.contains_key(entry.key()) { + d.blobs_only_a.push(entry.key().clone()); + } + } + for entry in b.blobs.iter() { + if !a.blobs.contains_key(entry.key()) { + d.blobs_only_b.push(entry.key().clone()); + } + } + + // Comms join. + for entry in a.comms.iter() { + match b.comms.get(entry.key()) { + None => d.comms_removed.push(entry.key().clone()), + Some(other) => { + if other.value() != entry.value() { + d.comms_changed.push(entry.key().clone()); + } + }, + } + } + for entry in b.comms.iter() { + if a.comms.get(entry.key()).is_none() { + d.comms_added.push(entry.key().clone()); + } + } + + // Header: main + assumptions. + if a.main != b.main { + d.main_changed = Some((a.main.clone(), b.main.clone())); + } + for addr in &a.assumptions { + if !b.assumptions.contains(addr) { + d.assumptions_removed.push(addr.clone()); + } + } + for addr in &b.assumptions { + if !a.assumptions.contains(addr) { + d.assumptions_added.push(addr.clone()); + } + } + + // Hints, joined on constants present in both envs. + let shared = + |addr: &Address| a.consts.contains_key(addr) && b.consts.contains_key(addr); + for (addr, ha) in &a.anon_hints { + if !shared(addr) { + continue; + } + let hb = b.anon_hints.get(addr); + if hb != Some(ha) { + d.hints_changed.push(( + addr.clone(), + hint_label(Some(ha)), + hint_label(hb), + )); + } + } + for (addr, hb) in &b.anon_hints { + if shared(addr) && !a.anon_hints.contains_key(addr) { + d.hints_changed.push(( + addr.clone(), + hint_label(None), + hint_label(Some(hb)), + )); + } + } + + d.consts_only_a.sort_unstable(); + d.consts_only_b.sort_unstable(); + d.blobs_only_a.sort_unstable(); + d.blobs_only_b.sort_unstable(); + d.comms_added.sort_unstable(); + d.comms_removed.sort_unstable(); + d.comms_changed.sort_unstable(); + d.assumptions_added.sort_unstable(); + d.assumptions_removed.sort_unstable(); + d.hints_changed.sort_unstable_by(|x, y| x.0.cmp(&y.0)); + + Ok(d) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::comm::Comm; + use crate::constant::{DefKind, defn_proj_constant}; + use crate::metadata::{ConstantMeta, ConstantMetaInfo, ExprMeta}; + use ix_common::env::DefinitionSafety; + use std::sync::Arc; + + fn n(s: &str) -> Name { + Name::str(Name::anon(), s.to_string()) + } + + /// Store `c` at its true content address; returns the address. + fn store_canonical(env: &Env, c: &Constant) -> Address { + let (addr, _) = c.commit(); + env.store_const(addr.clone(), c.clone()); + addr + } + + fn mk_defn(lvls: u64, typ: Arc, value: Arc) -> Definition { + Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls, + typ, + value, + } + } + + fn defn_c(typ: Arc, value: Arc) -> Constant { + Constant::new(ConstantInfo::Defn(mk_defn(0, typ, value))) + } + + fn defn_ct( + typ: Arc, + value: Arc, + sharing: Vec>, + refs: Vec
, + univs: Vec>, + ) -> Constant { + Constant::with_tables( + ConstantInfo::Defn(mk_defn(0, typ, value)), + sharing, + refs, + univs, + ) + } + + /// Single-constant env: store `c` canonically and name it `name`. + fn env1(name: &str, c: &Constant) -> (Env, Address) { + let env = Env::new(); + let addr = store_canonical(&env, c); + env.register_name(n(name), Named::with_addr(addr.clone())); + (env, addr) + } + + fn labels(v: &[String]) -> Vec<&str> { + v.iter().map(String::as_str).collect() + } + + fn diff(a: &Env, b: &Env) -> EnvDiff { + diff_envs(a, b, false).expect("diff_envs failed") + } + + fn diff_meta(a: &Env, b: &Env) -> EnvDiff { + diff_envs(a, b, true).expect("diff_envs failed") + } + + #[test] + fn identical_envs_empty_diff() { + let mut env = Env::new(); + let c = defn_c(Expr::var(3), Expr::var(0)); + let addr = store_canonical(&env, &c); + env.register_name(n("Foo"), Named::with_addr(addr.clone())); + let blob = env.store_blob(vec![1, 2, 3]); + env.store_comm( + Address::hash(b"comm"), + Comm::new(Address::hash(b"s"), Address::hash(b"p")), + ); + env.main = Some(addr.clone()); + env.assumptions.insert(blob); + env.anon_hints.insert(addr, ReducibilityHints::Regular(2)); + let copy = env.clone(); + let d = diff(&env, ©); + assert!(d.is_empty(), "anon self-diff should be empty: {d:?}"); + assert_eq!(d.stats_a, d.stats_b); + assert_eq!(d.stats_a.consts, 1); + assert!(diff_meta(&env, ©).is_empty(), "meta self-diff not empty"); + } + + #[test] + fn added_and_removed_name() { + let (a, _) = env1("Base", &defn_c(Expr::var(3), Expr::var(0))); + let b = a.clone(); + let extra = defn_c(Expr::var(4), Expr::var(1)); + let extra_addr = store_canonical(&b, &extra); + b.register_name(n("Extra"), Named::with_addr(extra_addr.clone())); + + let d = diff(&a, &b); + assert_eq!(d.named_added, vec![("Extra".to_string(), extra_addr.clone())]); + assert!(d.named_removed.is_empty() && d.named_changed.is_empty()); + assert_eq!(d.consts_only_b, vec![extra_addr.clone()]); + assert!(d.consts_only_a.is_empty()); + + let d = diff(&b, &a); + assert_eq!( + d.named_removed, + vec![("Extra".to_string(), extra_addr.clone())] + ); + assert_eq!(d.consts_only_a, vec![extra_addr]); + } + + #[test] + fn defn_value_only_changed() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(1))); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 1); + let c = &d.named_changed[0]; + assert_eq!(c.name, "Foo"); + assert_eq!((c.old_kind, c.new_kind), ("defn", "defn")); + assert_eq!(labels(&c.fields), ["value"]); + assert!(c.meta_fields.is_empty()); + } + + #[test] + fn defn_type_only_changed() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(4), Expr::var(0))); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + } + + #[test] + fn defn_scalars_changed() { + let ca = Constant::new(ConstantInfo::Defn(Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Safe, + lvls: 0, + typ: Expr::var(3), + value: Expr::var(0), + })); + let cb = Constant::new(ConstantInfo::Defn(Definition { + kind: DefKind::Definition, + safety: DefinitionSafety::Partial, + lvls: 1, + typ: Expr::var(3), + value: Expr::var(0), + })); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["safety", "lvls"]); + } + + /// Identical expr bytes over different refs tables are DIFFERENT. + #[test] + fn refs_shift_same_bytes_is_different() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let mk = |r: Address| { + defn_ct(Expr::var(9), Expr::reference(0, vec![]), vec![], vec![r], vec![]) + }; + let (a, _) = env1("Foo", &mk(x)); + let (b, _) = env1("Foo", &mk(y)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["value"]); + } + + /// Different indices resolving to the same addresses with the same + /// structure are EQUAL — a real type change is the only field. + #[test] + fn refs_permuted_same_resolution_is_equal() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(2), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y.clone(), x.clone()], + vec![], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + + // Pure permutation: no semantic field difference at all. + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y, x], + vec![], + ); + let (a, addr_a) = env1("Foo", &ca); + let (b, addr_b) = env1("Foo", &cb); + assert_ne!(addr_a, addr_b, "permuted tables must change the address"); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + } + + /// Sharing-table indirection vs inlined subterms: resolved-equal. + #[test] + fn share_vs_inline_equal() { + let t = Expr::all(Expr::var(7), Expr::var(0)); + let ca = defn_ct( + Expr::var(1), + Expr::app(Arc::new(Expr::Share(0)), Arc::new(Expr::Share(0))), + vec![t.clone()], + vec![], + vec![], + ); + let cb = + defn_ct(Expr::var(1), Expr::app(t.clone(), t), vec![], vec![], vec![]); + let (a, addr_a) = env1("Foo", &ca); + let (b, addr_b) = env1("Foo", &cb); + assert_ne!(addr_a, addr_b); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + } + + /// Univ-table shifts with identical resolution are not a type change; + /// changed univ content behind the same index is. + #[test] + fn univ_table_shift_equal_content_change_not() { + let ca = + defn_ct(Expr::sort(0), Expr::var(0), vec![], vec![], vec![Univ::zero()]); + let cb = defn_ct( + Expr::sort(1), + Expr::var(0), + vec![], + vec![], + vec![Univ::succ(Univ::zero()), Univ::zero()], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["encoding"]); + + let ca = + defn_ct(Expr::sort(0), Expr::var(0), vec![], vec![], vec![Univ::zero()]); + let cb = defn_ct( + Expr::sort(0), + Expr::var(0), + vec![], + vec![], + vec![Univ::succ(Univ::zero())], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type"]); + } + + /// A field-compare failure must not leave stale memo entries that + /// mask a later field's difference (type and value share Arcs here). + #[test] + fn memo_cleared_after_unequal_field() { + let e_a = Expr::app(Expr::var(1), Expr::var(5)); + let e_b = Expr::app(Expr::var(1), Expr::var(6)); + let ca = defn_c(e_a.clone(), e_a); + let cb = defn_c(e_b.clone(), e_b); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["type", "value"]); + } + + #[test] + fn metadata_only_change() { + let c = defn_c(Expr::var(3), Expr::var(0)); + let (a, addr) = env1("Foo", &c); + let b = Env::new(); + store_canonical(&b, &c); + let meta = ConstantMeta { + meta_refs: vec![Address::hash(b"extra-ref")], + ..ConstantMeta::default() + }; + b.register_name(n("Foo"), Named::new(addr.clone(), meta)); + + let d = diff(&a, &b); + assert!(d.is_empty(), "anon mode must ignore metadata: {d:?}"); + let d = diff_meta(&a, &b); + assert_eq!(d.named_meta_only.len(), 1); + assert_eq!(d.named_meta_only[0].0, "Foo"); + assert_eq!(labels(&d.named_meta_only[0].1), ["meta.refs"]); + assert!(d.named_changed.is_empty()); + + // original added + info kind change carry their own labels. + let b2 = Env::new(); + store_canonical(&b2, &c); + let info = ConstantMetaInfo::Def { + name: Address::hash(b"nm"), + lvls: vec![], + hints: ReducibilityHints::Regular(1), + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }; + let mut named2 = Named::new(addr.clone(), ConstantMeta::new(info)); + named2.set_original(addr.clone(), ConstantMeta::default()); + b2.register_name(n("Foo"), named2); + let d = diff_meta(&a, &b2); + assert_eq!( + labels(&d.named_meta_only[0].1), + ["meta.info(empty→def)", "original.added"] + ); + } + + #[test] + fn kind_change() { + let ca = Constant::new(ConstantInfo::Axio(crate::constant::Axiom { + is_unsafe: false, + lvls: 0, + typ: Expr::var(3), + })); + let cb = defn_c(Expr::var(3), Expr::var(0)); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + let c = &d.named_changed[0]; + assert_eq!((c.old_kind, c.new_kind), ("axio", "defn")); + assert_eq!(labels(&c.fields), ["kind"]); + } + + #[test] + fn main_assumptions_comms_hints() { + let shared = defn_c(Expr::var(3), Expr::var(0)); + let p = Address::hash(b"p"); + let q = Address::hash(b"q"); + let r = Address::hash(b"r"); + let c1 = Address::hash(b"comm1"); + let c2 = Address::hash(b"comm2"); + + let mut a = Env::new(); + let addr = store_canonical(&a, &shared); + a.main = Some(addr.clone()); + a.assumptions.insert(p.clone()); + a.assumptions.insert(q.clone()); + a.store_comm( + c1.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay1")), + ); + a.anon_hints.insert(addr.clone(), ReducibilityHints::Regular(1)); + + let mut b = Env::new(); + store_canonical(&b, &shared); + b.main = Some(p.clone()); + b.assumptions.insert(q); + b.assumptions.insert(r.clone()); + b.store_comm( + c1.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay2")), + ); + b.store_comm( + c2.clone(), + Comm::new(Address::hash(b"s"), Address::hash(b"pay3")), + ); + b.anon_hints.insert(addr.clone(), ReducibilityHints::Regular(2)); + + let d = diff(&a, &b); + assert_eq!(d.main_changed, Some((Some(addr.clone()), Some(p.clone())))); + assert_eq!(d.assumptions_added, vec![r]); + assert_eq!(d.assumptions_removed, vec![p]); + assert_eq!(d.comms_changed, vec![c1]); + assert_eq!(d.comms_added, vec![c2]); + assert!(d.comms_removed.is_empty()); + assert_eq!( + d.hints_changed, + vec![(addr, "regular(1)".to_string(), "regular(2)".to_string())] + ); + } + + /// One-level block descent: the changed member gets `block.*` detail, + /// the untouched sibling gets `block-siblings`. + #[test] + fn projection_block_descent() { + let f = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(0))); + let g_old = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(1))); + let g_new = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(2))); + let block_a = Constant::new(ConstantInfo::Muts(vec![f.clone(), g_old])); + let block_b = Constant::new(ConstantInfo::Muts(vec![f, g_new])); + + let a = Env::new(); + let block_a_addr = store_canonical(&a, &block_a); + let fa = store_canonical(&a, &defn_proj_constant(0, block_a_addr.clone())); + let ga = store_canonical(&a, &defn_proj_constant(1, block_a_addr)); + a.register_name(n("M.f"), Named::with_addr(fa)); + a.register_name(n("M.g"), Named::with_addr(ga)); + + let b = Env::new(); + let block_b_addr = store_canonical(&b, &block_b); + let fb = store_canonical(&b, &defn_proj_constant(0, block_b_addr.clone())); + let gb = store_canonical(&b, &defn_proj_constant(1, block_b_addr)); + b.register_name(n("M.f"), Named::with_addr(fb)); + b.register_name(n("M.g"), Named::with_addr(gb)); + + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 2); + let mf = &d.named_changed[0]; + let mg = &d.named_changed[1]; + assert_eq!((mf.name.as_str(), mg.name.as_str()), ("M.f", "M.g")); + assert_eq!((mf.old_kind, mf.new_kind), ("dprj", "dprj")); + assert_eq!(labels(&mf.fields), ["block-siblings"]); + assert_eq!(labels(&mg.fields), ["block.value"]); + } + + #[test] + fn recursor_rules_changed() { + let mk = |rhs: Arc, rules_extra: bool| { + let mut rules = vec![crate::constant::RecursorRule { fields: 2, rhs }]; + if rules_extra { + rules + .push(crate::constant::RecursorRule { fields: 0, rhs: Expr::var(0) }); + } + Constant::new(ConstantInfo::Recr(Recursor { + k: false, + is_unsafe: false, + lvls: 1, + params: 1, + indices: 0, + motives: 1, + minors: 1, + typ: Expr::var(9), + rules, + })) + }; + let (a, _) = env1("R", &mk(Expr::var(4), false)); + let (b, _) = env1("R", &mk(Expr::var(5), false)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["rules[0].rhs"]); + + let (a, _) = env1("R", &mk(Expr::var(4), false)); + let (b, _) = env1("R", &mk(Expr::var(4), true)); + let d = diff(&a, &b); + assert_eq!(labels(&d.named_changed[0].fields), ["rules.len"]); + } +} diff --git a/crates/ixon/src/lib.rs b/crates/ixon/src/lib.rs index 84a13addb..b6327ee04 100644 --- a/crates/ixon/src/lib.rs +++ b/crates/ixon/src/lib.rs @@ -9,6 +9,8 @@ pub mod assumption_tree; pub mod comm; pub mod constant; +#[cfg(not(target_arch = "riscv64"))] +pub mod diff; pub mod env; pub mod error; pub mod expr; @@ -29,6 +31,8 @@ pub use constant::{ Definition, DefinitionProj, Inductive, InductiveProj, MutConst, Quotient, Recursor, RecursorProj, RecursorRule, }; +#[cfg(not(target_arch = "riscv64"))] +pub use diff::{EnvDiff, EnvStats, NamedChange, diff_envs}; pub use env::{Env, Named}; pub use error::{CompileError, DecompileError, SerializeError}; pub use expr::Expr; From 63f33bfe3fd114ddee7b2d49740d42794278253c Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Thu, 9 Jul 2026 21:42:19 -0400 Subject: [PATCH 3/8] Ixon: ix diff progress reporting + memory-lean anon loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit diff_envs_with adds a JoinProgress callback (fires every 100k named-join entries and once at completion); diff_envs delegates with a no-op. rs_diff_envs reports phases to stderr ([rs_diff_envs] ..., the rs_compile_env idiom): per-env parse timings with reader label and live join progress — emitted only when an input is >= 100 MB, so unit tests, property tests, and small bundles stay silent. Anon mode (the default) now loads both envs via parse_lazy_index + Env::from_lazy_index — constant byte-windows, name->addr, verbatim §3 hints, §6 comms — never materializing ConstantMeta. Two full-reader mathlib envs OOM a 124 GiB machine; the lazy path diffs two 3.2 GB mathlib envs in ~62 s at 54 GiB peak with identical output (locked by a lazy/full parity test). --meta keeps the full reader, since Named.meta/original are the point of that mode. parse_lazy_index now carries §3 hints and §6 comms verbatim on LazyIndex (both tiny) and, since it consumes every section, enforces EOF exactly like Env::get. Mathlib-scale validation (v4.29.0 tag vs 8850ed93, 5 days of master): 3763 added / 1522 removed / 143528 changed names over 744k joined, exit 1, deterministic — zero [encoding] entries; 5 genuine kind changes and 1 hint change surface above the content-address ripple. --- crates/ffi/src/lean_ixon/diff.rs | 84 ++++++++++++++++--- crates/ixon/src/diff.rs | 136 ++++++++++++++++++++++++++++++- crates/ixon/src/env.rs | 54 ++++++++++++ crates/ixon/src/lib.rs | 4 +- crates/ixon/src/serialize.rs | 37 +++++++-- 5 files changed, 295 insertions(+), 20 deletions(-) diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs index 62b2c7e3c..8f0b7f875 100644 --- a/crates/ffi/src/lean_ixon/diff.rs +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -1,13 +1,20 @@ -//! `rs_diff_envs`: parse two serialized envs with the full reader -//! (`Env::get`, preserving `Named.original`), diff them in Rust -//! (`ixon::diff::diff_envs`), and marshal the [`EnvDiff`] report to -//! Lean (`Ixon.EnvDiff`). +//! `rs_diff_envs`: parse two serialized envs, diff them in Rust +//! (`ixon::diff`), and marshal the [`EnvDiff`] report to Lean +//! (`Ixon.EnvDiff`). +//! +//! Anon mode (the default) loads via the memory-lean lazy-index path +//! (`parse_lazy_index` + `Env::from_lazy_index` — ConstantMeta is never +//! materialized); meta mode uses the full reader (`Env::get`, +//! preserving `Named.meta`/`original`). Large inputs get incremental +//! progress on stderr. //! //! Names cross the boundary pre-rendered (`Name::pretty()`) and //! pre-sorted; addresses cross raw so Lean owns display formatting. +use std::time::Instant; + use ix_common::address::Address; -use ixon::diff::{EnvDiff, EnvStats, NamedChange, diff_envs}; +use ixon::diff::{EnvDiff, EnvStats, NamedChange, diff_envs_with}; use ixon::env::Env as IxonEnv; use lean_ffi::object::{ LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, @@ -137,18 +144,58 @@ impl LeanIxonEnvDiff { } } +/// Inputs at least this large get incremental progress on stderr +/// (`[rs_diff_envs] …` lines, matching the `[rs_compile_env]` idiom). +/// Small inputs — unit tests, property tests, tiny bundles — stay +/// silent. +const PROGRESS_MIN_BYTES: usize = 100 * 1024 * 1024; + /// FFI: diff two serialized envs. /// `(a b : ByteArray) → (meta : Bool) → Except String Ixon.EnvDiff`. -/// Pure; both buffers are parsed with the full reader (`Env::get`). +/// Pure result; reader choice (lazy-index vs full) follows `meta` — +/// see the module docs. For large inputs, progress goes to stderr. #[unsafe(no_mangle)] pub extern "C" fn rs_diff_envs( a: LeanByteArray>, b: LeanByteArray>, meta: LeanBool>, ) -> LeanExcept { + let progress = a.as_bytes().len() >= PROGRESS_MIN_BYTES + || b.as_bytes().len() >= PROGRESS_MIN_BYTES; + let want_meta = meta.to_bool(); + // Anon mode loads via the lazy index: constant byte-windows + + // name→addr + §3 hints + §6 comms, never materializing ConstantMeta + // — at mathlib scale the full reader's metadata costs tens of GB and + // OOMs a 128 GB machine when two envs are held at once. Meta mode + // needs `Named.meta`/`original`, so it pays for the full reader. let parse = |bytes: &[u8], which: &str| -> Result { - let mut cursor: &[u8] = bytes; - IxonEnv::get(&mut cursor).map_err(|e| format!("{which} input: {e}")) + let reader = if want_meta { "full" } else { "anon" }; + if progress { + eprintln!( + "[rs_diff_envs] parsing {which} env ({} MB, {reader} reader)...", + bytes.len() / 1_000_000 + ); + } + let t = Instant::now(); + let env = if want_meta { + let mut cursor: &[u8] = bytes; + IxonEnv::get(&mut cursor).map_err(|e| format!("{which} input: {e}"))? + } else { + let index = IxonEnv::parse_lazy_index(bytes) + .map_err(|e| format!("{which} input: {e}"))?; + IxonEnv::from_lazy_index(&index, bytes) + .map_err(|e| format!("{which} input: {e}"))? + }; + if progress { + eprintln!( + "[rs_diff_envs] {which} env parsed in {:.1}s ({} consts, {} named, {} blobs)", + t.elapsed().as_secs_f64(), + env.consts.len(), + env.named.len(), + env.blobs.len() + ); + } + Ok(env) }; let env_a = match parse(a.as_bytes(), "first") { Ok(env) => env, @@ -158,8 +205,25 @@ pub extern "C" fn rs_diff_envs( Ok(env) => env, Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), }; - match diff_envs(&env_a, &env_b, meta.to_bool()) { - Ok(d) => LeanExcept::ok(LeanIxonEnvDiff::build(&d)), + let t = Instant::now(); + let mut on_progress = |p: ixon::diff::JoinProgress| { + if progress { + eprintln!( + "[rs_diff_envs] named join: {}/{} ({} changed so far)", + p.done, p.total, p.changed + ); + } + }; + match diff_envs_with(&env_a, &env_b, want_meta, &mut on_progress) { + Ok(d) => { + if progress { + eprintln!( + "[rs_diff_envs] diff computed in {:.1}s", + t.elapsed().as_secs_f64() + ); + } + LeanExcept::ok(LeanIxonEnvDiff::build(&d)) + }, Err(e) => LeanExcept::error_string(&format!("rs_diff_envs: {e}")), } } diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs index e7045d883..5307ffa58 100644 --- a/crates/ixon/src/diff.rs +++ b/crates/ixon/src/diff.rs @@ -56,7 +56,7 @@ pub struct EnvStats { } /// One name present in both envs whose constant address changed. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct NamedChange { /// `Name::pretty()` of the joined name. pub name: String, @@ -77,7 +77,7 @@ pub struct NamedChange { /// Report produced by [`diff_envs`]. All name-keyed vectors are sorted /// by (pretty name, name hash); address vectors are sorted ascending. /// Set-difference lists are complete — display layers cap as needed. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct EnvDiff { /// `None` = unchanged, otherwise `(a.main, b.main)`. pub main_changed: Option<(Option
, Option
)>, @@ -771,10 +771,39 @@ fn sort_by_name(v: &mut [(String, Name, T)]) { v.sort_by(|x, y| x.0.cmp(&y.0).then_with(|| x.1.cmp(&y.1))); } +/// Coarse progress for the named join — the only diff phase whose cost +/// scales with the number of *changed* names (each one materializes two +/// constants and walks their expressions). Parses happen before +/// [`diff_envs`] and are the caller's to report. +#[derive(Clone, Copy, Debug)] +pub struct JoinProgress { + /// Entries of the first env's named map processed so far. + pub done: usize, + /// Total entries in the first env's named map. + pub total: usize, + /// Changed names (addr differs) classified so far. + pub changed: usize, +} + +/// Named-join entries between progress callbacks in [`diff_envs_with`]. +const JOIN_PROGRESS_STRIDE: usize = 100_000; + /// Diff two environments. `meta = false` (anon mode, the default) /// compares only anonymous structure; `meta = true` additionally /// compares `Named` metadata content. See the module docs. pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { + diff_envs_with(a, b, meta, &mut |_| {}) +} + +/// [`diff_envs`] with a progress callback: `on_progress` fires every +/// `JOIN_PROGRESS_STRIDE` named-join entries and once when the join +/// completes (so the final event always carries `done == total`). +pub fn diff_envs_with( + a: &Env, + b: &Env, + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { let mut d = EnvDiff { stats_a: env_stats(a), stats_b: env_stats(b), @@ -788,6 +817,8 @@ pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { let mut removed: Vec<(String, Name, Address)> = Vec::new(); let mut changed: Vec<(String, Name, NamedChange)> = Vec::new(); let mut meta_only: Vec<(String, Name, Vec)> = Vec::new(); + let join_total = a.named.len(); + let mut join_done: usize = 0; for entry in a.named.iter() { let (name, na) = (entry.key(), entry.value()); match b.named.get(name) { @@ -807,7 +838,21 @@ pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { } }, } + join_done += 1; + if join_done.is_multiple_of(JOIN_PROGRESS_STRIDE) && join_done < join_total + { + on_progress(JoinProgress { + done: join_done, + total: join_total, + changed: changed.len(), + }); + } } + on_progress(JoinProgress { + done: join_total, + total: join_total, + changed: changed.len(), + }); for entry in b.named.iter() { let name = entry.key(); if a.named.get(name).is_none() { @@ -1342,6 +1387,93 @@ mod tests { assert_eq!(labels(&mg.fields), ["block.value"]); } + /// Anon-mode diffs must be identical whether the envs came from the + /// full reader or the lazy-index path (`ix diff`'s memory-lean route). + #[test] + fn lazy_index_env_matches_full_reader_in_anon_mode() { + let build = |value: Arc, hint: u32, with_extra: bool| { + let mut env = Env::new(); + let register = |env: &Env, label: &str, addr: Address| { + let name = n(label); + env + .names + .insert(Address::from_blake3_hash(*name.get_hash()), name.clone()); + env.register_name(name, Named::with_addr(addr)); + }; + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + register(&env, "Foo", addr.clone()); + // Hint lives on a constant PRESENT IN BOTH envs — the hints diff + // joins on shared consts only. + let stable = defn_c(Expr::var(7), Expr::var(0)); + let stable_addr = store_canonical(&env, &stable); + register(&env, "Stable", stable_addr.clone()); + env.anon_hints.insert(stable_addr, ReducibilityHints::Regular(hint)); + env.store_blob(vec![1, 2, 3]); + env.store_comm( + Address::hash(b"comm"), + Comm::new(Address::hash(b"s"), Address::hash(b"p")), + ); + env.main = Some(addr); + env.assumptions.insert(Address::hash(b"assumed")); + if with_extra { + let extra = defn_c(Expr::var(5), Expr::var(2)); + let extra_addr = store_canonical(&env, &extra); + register(&env, "Extra", extra_addr); + } + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let bytes_a = build(Expr::var(0), 1, false); + let bytes_b = build(Expr::var(1), 2, true); + + let full = |bytes: &[u8]| { + let mut cursor = bytes; + Env::get(&mut cursor).expect("full read failed") + }; + let lazy = |bytes: &[u8]| { + let index = Env::parse_lazy_index(bytes).expect("lazy index failed"); + Env::from_lazy_index(&index, bytes).expect("from_lazy_index failed") + }; + + let via_full = diff(&full(&bytes_a), &full(&bytes_b)); + let via_lazy = diff(&lazy(&bytes_a), &lazy(&bytes_b)); + assert_eq!(via_full, via_lazy); + // Sanity: the pair genuinely exercises every section. + assert_eq!(via_full.named_changed.len(), 1); + assert_eq!(via_full.named_added.len(), 1); + assert_eq!(via_full.hints_changed.len(), 1); + // Lazy self-diff is empty. + assert!(diff(&lazy(&bytes_a), &lazy(&bytes_a)).is_empty()); + } + + #[test] + fn join_progress_final_event_totals() { + let (a, _) = env1("Base", &defn_c(Expr::var(3), Expr::var(0))); + let b = a.clone(); + let extra = defn_c(Expr::var(4), Expr::var(1)); + let extra_addr = store_canonical(&b, &extra); + b.register_name(n("Extra"), Named::with_addr(extra_addr)); + // Change Base's constant so exactly one join entry classifies. + let changed = defn_c(Expr::var(3), Expr::var(9)); + let changed_addr = store_canonical(&b, &changed); + b.register_name(n("Base"), Named::with_addr(changed_addr)); + + let mut events: Vec = Vec::new(); + let d = diff_envs_with(&a, &b, false, &mut |p| events.push(p)) + .expect("diff_envs_with failed"); + let last = events.last().expect("no progress events"); + assert_eq!(last.done, last.total); + assert_eq!(last.total, a.named.len()); + assert_eq!(last.changed, d.named_changed.len()); + assert_eq!(d.named_changed.len(), 1); + assert!( + events.windows(2).all(|w| w[0].done <= w[1].done), + "progress must be monotone" + ); + } + #[test] fn recursor_rules_changed() { let mk = |rhs: Arc, rules_extra: bool| { diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index c17fb9010..dae1b7f1a 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -203,6 +203,11 @@ pub struct LazyIndex { pub main: Option
, /// Bundle trust boundary (`Env::assumptions`), in header (sorted) order. pub assumptions: Vec
, + /// §3 anon_hints verbatim (file order) — the same content the full + /// reader puts in `Env::anon_hints`. + pub hints: Vec<(Address, ReducibilityHints)>, + /// §6 comms (tiny in practice; empty for compile-produced envs). + pub comms: Vec<(Address, Comm)>, } /// The Ixon environment. @@ -352,6 +357,55 @@ impl Env { self.consts.insert(addr, LazyConstant::from_mmap_slice(mmap, offset, len)); } + /// Build a metadata-light `Env` from a parsed [`LazyIndex`] over + /// `data` — the exact buffer handed to [`Env::parse_lazy_index`] + /// (constant windows are copied out of it, so the result owns its + /// bytes). `named` entries carry empty `ConstantMeta`; everything an + /// anon-mode consumer needs (consts, name→addr, §3 hints, blobs, + /// comms, `main`, `assumptions`) is populated. + /// + /// This is the memory-lean anon loading path (e.g. `ix diff`): the + /// full reader materializes every `ConstantMeta`, which at mathlib + /// scale costs tens of GB; this path costs roughly the consts + /// section plus the name table. Host-only — see `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn from_lazy_index( + index: &LazyIndex, + data: &[u8], + ) -> Result { + let mut env = Env::new(); + for c in &index.consts { + let end = c + .offset + .checked_add(c.len) + .filter(|e| *e <= data.len()) + .ok_or_else(|| { + format!( + "from_lazy_index: constant window [{}, +{}) out of bounds ({} bytes)", + c.offset, + c.len, + data.len() + ) + })?; + env.store_const_lazy(c.addr.clone(), Arc::from(&data[c.offset..end])); + } + for n in &index.named { + env.register_name(n.name.clone(), Named::with_addr(n.addr.clone())); + } + for (addr, hint) in &index.hints { + env.anon_hints.insert(addr.clone(), *hint); + } + for (addr, bytes) in &index.blobs { + env.blobs.insert(addr.clone(), bytes.clone()); + } + for (addr, comm) in &index.comms { + env.comms.insert(addr.clone(), comm.clone()); + } + env.main = index.main.clone(); + env.assumptions = index.assumptions.iter().cloned().collect(); + Ok(env) + } + /// Get a constant by address, materializing on demand. /// /// Returns `None` if the address is not present or materialization diff --git a/crates/ixon/src/lib.rs b/crates/ixon/src/lib.rs index b6327ee04..896e7a956 100644 --- a/crates/ixon/src/lib.rs +++ b/crates/ixon/src/lib.rs @@ -32,7 +32,9 @@ pub use constant::{ Recursor, RecursorProj, RecursorRule, }; #[cfg(not(target_arch = "riscv64"))] -pub use diff::{EnvDiff, EnvStats, NamedChange, diff_envs}; +pub use diff::{ + EnvDiff, EnvStats, JoinProgress, NamedChange, diff_envs, diff_envs_with, +}; pub use env::{Env, Named}; pub use error::{CompileError, DecompileError, SerializeError}; pub use expr::Expr; diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index 0d5da7cf6..efe2121f9 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -1707,9 +1707,9 @@ impl Env { /// Deserialize an Env from bytes. /// /// The whole buffer must be consumed — trailing bytes after the - /// final section are rejected (truncation/concatenation guard). The - /// early-stop readers (`get_anon`, `get_anon_mmap`, - /// `parse_lazy_index`) cannot make that check by design. + /// final section are rejected (truncation/concatenation guard). + /// `parse_lazy_index` enforces the same; the early-stop readers + /// (`get_anon`, `get_anon_mmap`) cannot make that check by design. pub fn get(buf: &mut &[u8]) -> Result { // Header: tag + stored merkle root (verified at the end against // the recomputed root; empty const sets store `zero_address()`) + @@ -1847,10 +1847,14 @@ impl Env { /// - the `names` table and each `Named`'s `ExprMetaArena` are parsed only to /// advance the cursor and are then dropped, keeping just `name → addr` and /// the per-`Defn` reducibility hint; - /// - `comms` are skipped entirely. + /// - §3 hints and §6 comms are carried verbatim (both tiny), so an `Env` + /// rebuilt from the index (see [`Env::from_lazy_index`]) matches the full + /// reader on every anon-visible section. /// /// `data` must be the whole buffer (offsets are relative to its start). The - /// env-level merkle root over const addresses is still re-verified. + /// env-level merkle root over const addresses is still re-verified. Since + /// this reader now consumes every section, trailing bytes are rejected + /// exactly as in [`Env::get`]. pub fn parse_lazy_index(data: &[u8]) -> Result { let mut buf: &[u8] = data; @@ -1885,8 +1889,10 @@ impl Env { // Section 3: anon_hints — the canonical hint channel (the writer // always emits it, deriving from Named metadata when needed), so // `LazyNamed.hint` is a plain lookup instead of a metadata harvest. + // Kept verbatim on the index for full-reader parity. + index.hints = read_hints_section(&mut buf)?; let hints_map: FxHashMap = - read_hints_section(&mut buf)?.into_iter().collect(); + index.hints.iter().cloned().collect(); // Section 4: Names — parsed to build the index for metadata decoding, then // dropped (the check never consults the name component table). @@ -1917,7 +1923,24 @@ impl Env { index.named.push(LazyNamed { name, addr: named.addr, hint }); } - // Section 6 (comms) is never reached: not needed by the check path. + // Section 6: Comms — carried verbatim (tiny; empty for + // compile-produced envs). The check path ignores them, but + // `Env::from_lazy_index` consumers (e.g. `ix diff`) want parity + // with the full reader. + let num_comms = get_u64(&mut buf)?; + for _ in 0..num_comms { + let addr = get_address(&mut buf)?; + let comm = Comm::get(&mut buf)?; + index.comms.push((addr, comm)); + } + + // Every section consumed — enforce EOF like `Env::get`. + if !buf.is_empty() { + return Err(format!( + "parse_lazy_index: {} trailing bytes after final section", + buf.len() + )); + } // Re-verify the merkle root over const addresses (header integrity). let mut const_addrs: Vec
= From 903a96121a7d6bf69db0d2adf082770bbb151b28 Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Fri, 10 Jul 2026 00:31:50 -0400 Subject: [PATCH 4/8] =?UTF-8?q?Ixon:=20ix=20pack=20=E2=80=94=20bundle-prod?= =?UTF-8?q?ucing=20CLI=20over=20prune=5Fto=5Fclosure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lake exe ix pack ` prunes a serialized env to the self-contained bundle pinning one named constant and writes it as a standalone .ixe — the first production consumer of Env::prune_to_closure (sets main, records reached cut-points in assumptions, carries display metadata to fixpoint) followed by the receiver-side Env::validate_closed. - New FFI rs_pack_env (crates/ffi/src/lean_ixon/pack.rs), mirroring the rs_env_extract read→resolve→write shape: full reader, displayed name→address resolution over env.named, --assume entries resolving as names first then 64-hex addresses. - New Ix/Cli/PackCmd.lean: --assume/--assume-file (ConstsFile.gather), --out (default .ixe), --verbose; exit 0/1. - Tests/FFI/Ixon.lean: six directed IO tests over a temp-file fixture (closure bundle, removals-only diff subset, name and hex assume cuts, unknown-root and main-assumed errors). The fixture stores name-string blobs via addNameComponentsWithBlobs — the compiler convention prune_to_closure recreates via carry_name. - docs/Ixon.md: pack workflow in the bundle section. E2E: Nat.add out of the 324 MB stdlib env → 13 KB closed bundle (17 consts, 43 blobs) in ~4 s; --assume Nat drops the cut constant and records 1 assumption. --- Ix/Cli/PackCmd.lean | 73 ++++++++++++++++ Ix/Ixon.lean | 14 +++ Main.lean | 2 + Tests/FFI/Ixon.lean | 118 ++++++++++++++++++++++++++ crates/ffi/src/lean_ixon.rs | 1 + crates/ffi/src/lean_ixon/pack.rs | 141 +++++++++++++++++++++++++++++++ docs/Ixon.md | 16 ++++ 7 files changed, 365 insertions(+) create mode 100644 Ix/Cli/PackCmd.lean create mode 100644 crates/ffi/src/lean_ixon/pack.rs diff --git a/Ix/Cli/PackCmd.lean b/Ix/Cli/PackCmd.lean new file mode 100644 index 000000000..bfad1b7cb --- /dev/null +++ b/Ix/Cli/PackCmd.lean @@ -0,0 +1,73 @@ +/- + `ix pack `: prune a serialized env to the self-contained + bundle pinning one named constant, and write it as a standalone `.ixe`. + + A bundle is an `.ixe` whose `main` points at a distinguished constant; + because a constant's address is a merkle root over its whole dependency + DAG, `main`'s 32 bytes alone pin the value — the bundle is the + data-availability artifact that ships the bytes. `--assume` declares + trust-boundary cut-points: reached cut-points are recorded in the + bundle's `assumptions` instead of being carried (thin bundles). + + The heavy lifting is `Env::prune_to_closure` (3-edge value closure of + `main`, display metadata carried to fixpoint) followed by + `Env::validate_closed` — the same check a receiver runs — so a written + bundle is closed by construction. + + Different from `ix shard extract`: extract produces a general sub-env + for the kernel-check pipeline (no `main`, no `assumptions`, anon-work + block closure); pack produces a verified bundle with a root and an + explicit trust boundary. +-/ +module +public import Cli +public import Ix.Ixon +public import Ix.Cli.ConstsFile + +public section + +namespace Ix.Cli.PackCmd + +def runPackCmd (p : Cli.Parsed) : IO UInt32 := do + let some pathArg := p.positionalArg? "path" + | p.printError "error: must specify to a source .ixe file" + return 1 + let envPath := pathArg.as! String + let some nameArg := p.positionalArg? "name" + | p.printError "error: must specify of the bundle root constant" + return 1 + let mainName := nameArg.as! String + let assume ← Ix.Cli.ConstsFile.gather p "assume" "assume-file" + let outPath : String := + match p.flag? "out" with + | some flag => flag.as! String + | none => s!"{mainName}.ixe" + let verbose := (p.flag? "verbose").isSome + try + Ixon.rsPackEnv envPath mainName assume outPath verbose + IO.println s!"[pack] wrote {outPath} (main {mainName}, \ + {assume.size} assumption cut(s) declared)" + return (0 : UInt32) + catch e => + IO.eprintln s!"error: {e.toString}" + return (1 : UInt32) + +end Ix.Cli.PackCmd + +open Ix.Cli.PackCmd in +def packCmd : Cli.Cmd := `[Cli| + pack VIA runPackCmd; + "Prune a `.ixe` env to the self-contained bundle pinning one constant (sets `main`; validated closed)" + + FLAGS: + assume : String; "Comma-separated cut-point constants — displayed names or 64-hex addresses. Reached cut-points are recorded in the bundle's `assumptions` instead of carried (thin bundle)." + "assume-file" : String; "Additionally read cut-points from a file (one per line; `#` comments and blank lines ignored). Unions with --assume." + out : String; "Output `.ixe` path. Defaults to `.ixe` (e.g. `Nat.add.ixe`)." + verbose; "Print pack details (source stats, kept counts, bytes written) to stderr." + + ARGS: + path : String; "Path to the source `.ixe` (e.g. from `ix compile`)." + name : String; "Displayed name of the bundle root constant (e.g. `Nat.add`)." +] + +end diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index e1a5edb48..b48b40b32 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -2357,6 +2357,20 @@ def rsDiffEnvs (a b : ByteArray) (compareMeta : Bool := false) : Except String EnvDiff := rsDiffEnvsFFI a b compareMeta +/-! ### Env pack (`rs_pack_env`) -/ + +/-- Pack a value bundle in Rust: read the env at `envPath` (full + reader), resolve `mainName` (displayed form) to its constant + address, prune to the self-contained closure + (`Env::prune_to_closure` — sets `main`, collects reached cut-points + into `assumptions`, carries display metadata to fixpoint), validate + (`Env::validate_closed`), and write the bundle to `outPath`. + `assume` entries resolve as displayed names first, else as 64-hex + constant addresses. Failures surface as `IO` errors. -/ +@[extern "rs_pack_env"] +opaque rsPackEnv : @& String → @& String → @& Array String → @& String → + Bool → IO Unit + /-! ## Canonical merkle root over consts -/ @[extern "rs_env_merkle_root"] diff --git a/Main.lean b/Main.lean index 1ed9e0c07..c3bdbd4b5 100644 --- a/Main.lean +++ b/Main.lean @@ -8,6 +8,7 @@ import Ix.Cli.ClaimCmd import Ix.Cli.CompileCmd import Ix.Cli.DiffCmd import Ix.Cli.IngressCmd +import Ix.Cli.PackCmd import Ix.Cli.ProfileCmd import Ix.Cli.ProveCmd import Ix.Cli.ShardCmd @@ -33,6 +34,7 @@ def ixCmd : Cli.Cmd := `[Cli| checkRsCmd; claimCmd; diffCmd; + packCmd; treeCmd; profileCmd; proveCmd; diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index a62d687e7..44119ecb7 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -363,6 +363,122 @@ def selfDiffEmpty (compareMeta : Bool) (env : RawEnv) : Bool := | .ok d => d.isEmpty | .error _ => false +/-! ## Env pack FFI (`rs_pack_env`) + +The prune/validate engine is covered by Rust unit tests (the +`prune_to_closure` tests in `crates/ixon/src/env.rs`); these pin the +FFI surface end-to-end — file IO, name/hex resolution, bundle-field +population — and the closed-subset relationship to the source env. -/ + +/-- Write `env` to a temp `.ixe`, pack `mainName` with `assume` cuts, + and parse the resulting bundle. Pack failures surface as `.error`. -/ +private def packFixture (env : Env) (mainName : String) + (assume : Array String) : IO (Except String Env) := do + let dir ← IO.FS.createTempDir + let src := dir / "src.ixe" + let out := dir / "bundle.ixe" + -- Every failure mode is caught into `.error`, so cleanup always runs. + let result ← try + IO.FS.writeBinFile src (serEnv env) + Ixon.rsPackEnv src.toString mainName assume out.toString false + let bytes ← IO.FS.readBinFile out + pure (Ixon.rsDeEnv bytes) + catch e => + pure (.error e.toString) + IO.FS.removeDirAll dir + return result + +/-- IO test asserting on a successfully packed bundle. -/ +private def packTest (descr : String) (act : IO (Except String Env)) + (check : Env → Bool) : TestSeq := + .individualIO descr none (do + match ← act with + | .ok bundle => + let ok := check bundle + pure (ok, 0, 0, if ok then none else some "bundle assertion failed") + | .error e => pure (false, 0, 0, some e)) .done + +/-- IO test asserting the pack call fails. -/ +private def packErrTest (descr : String) (act : IO (Except String Env)) : + TestSeq := + .individualIO descr none (do + match ← act with + | .ok _ => pure (false, 0, 0, some "expected pack to fail") + | .error _ => pure (true, 0, 0, none)) .done + +def envPackTests : TestSeq := + let fooName := Ix.Name.mkStr Ix.Name.mkAnon "foo" + let barName := Ix.Name.mkStr Ix.Name.mkAnon "bar" + let quxName := Ix.Name.mkStr Ix.Name.mkAnon "qux" + let leaf (v : UInt64) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .var v } + sharing := #[], refs := #[], univs := #[] } + let fooConst := leaf 0 + let quxConst := leaf 1 + let fooAddr := Address.blake3 (serConstant fooConst) + -- `bar` reaches `foo` through its refs table (a genuine closure + -- edge); `qux` is unreachable from `bar` and must be pruned away. + let barConst : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .ref 0 #[] } + sharing := #[], refs := #[fooAddr], univs := #[] } + let barAddr := Address.blake3 (serConstant barConst) + let src : Env := Id.run do + let mut env : Env := {} + for (n, c) in [(fooName, fooConst), (barName, barConst), + (quxName, quxConst)] do + let addr := Address.blake3 (serConstant c) + env := env.storeConst addr c + -- String components ride as blobs (the compiler's convention); + -- prune's `carry_name` recreates them in the bundle, so a fixture + -- without them would spuriously diff as bundle-only blobs. + let (names, blobs) := + RawEnv.addNameComponentsWithBlobs env.names env.blobs n + env := { env with names, blobs } + env := env.registerName n { addr, constMeta := .empty } + return env + packTest "EnvPack: closure bundle pins main and prunes" + (packFixture src "bar" #[]) (fun b => + b.main == some barAddr + && b.getAddr? barName == some barAddr + && b.getAddr? fooName == some fooAddr + && (b.getAddr? quxName).isNone + && b.consts.size == 2 + && b.assumptions.isEmpty) ++ + (TestSeq.individualIO "EnvPack: bundle is a removals-only subset under diff" + none (do + match ← packFixture src "bar" #[] with + | .error e => pure (false, 0, 0, some e) + | .ok b => + match Ixon.rsDiffEnvs (serEnv src) (serEnv b) with + | .error e => pure (false, 0, 0, some s!"diff failed: {e}") + | .ok d => + let ok := d.mainChanged == some (none, some barAddr) + && d.namedRemoved.size == 1 + && d.namedAdded.isEmpty && d.namedChanged.isEmpty + && d.constsOnlyB.isEmpty && d.blobsOnlyB.isEmpty + let msg := s!"mainOk={d.mainChanged == some (none, some barAddr)} \ + removed={d.namedRemoved.map (·.1)} \ + added={d.namedAdded.map (·.1)} \ + changed={d.namedChanged.map (·.name)} \ + constsOnlyB={d.constsOnlyB.size} blobsOnlyB={d.blobsOnlyB.size}" + pure (ok, 0, 0, if ok then none else some msg)) .done) ++ + packTest "EnvPack: assume cut (by name) records assumption" + (packFixture src "bar" #["foo"]) (fun b => + b.main == some barAddr + && (b.getAddr? fooName).isNone + && b.consts.size == 1 + && b.assumptions.size == 1 + && b.assumptions.contains fooAddr) ++ + packTest "EnvPack: assume cut (by hex address) records assumption" + (packFixture src "bar" #[toString fooAddr]) (fun b => + b.consts.size == 1 && b.assumptions.contains fooAddr) ++ + packErrTest "EnvPack: unknown root name errors" + (packFixture src "nope" #[]) ++ + packErrTest "EnvPack: main cannot be assumed" + (packFixture src "bar" #["bar"]) + /-! ## Test Suite -/ def suite : List TestSeq := [ @@ -410,6 +526,8 @@ def suite : List TestSeq := [ (∀ env : RawEnv, selfDiffEmpty false env), checkIO "Ixon.EnvDiff self-diff empty (meta)" (∀ env : RawEnv, selfDiffEmpty true env), + ---- Env pack + envPackTests, ] end Tests.FFI.Ixon diff --git a/crates/ffi/src/lean_ixon.rs b/crates/ffi/src/lean_ixon.rs index 742167dda..8cc7a26e6 100644 --- a/crates/ffi/src/lean_ixon.rs +++ b/crates/ffi/src/lean_ixon.rs @@ -11,6 +11,7 @@ pub mod enums; pub mod env; pub mod expr; pub mod meta; +pub mod pack; #[cfg(feature = "test-ffi")] pub mod serialize; #[cfg(feature = "test-ffi")] diff --git a/crates/ffi/src/lean_ixon/pack.rs b/crates/ffi/src/lean_ixon/pack.rs new file mode 100644 index 000000000..00868ee7c --- /dev/null +++ b/crates/ffi/src/lean_ixon/pack.rs @@ -0,0 +1,141 @@ +//! `rs_pack_env`: read a serialized env, prune it to the self-contained +//! closure of one named constant ([`ixon::env::Env::prune_to_closure`] — +//! sets `main` and collects reached cut-points into `assumptions`), +//! validate the result ([`ixon::env::Env::validate_closed`]), and write +//! the bundle `.ixe`. +//! +//! Uses the full reader (`Env::get`): the prune's named fixpoint carries +//! display metadata (names, `DataValue` blobs, aux `original`s), which +//! the anon/lazy views drop. At mathlib scale that means tens of GB +//! resident for the source env — functional, but heavy; a leaner +//! selective-metadata pack (stream §5, carry only survivors) is future +//! work on the diff meta-sweep streaming machinery. + +use ix_common::address::Address; +use ixon::env::Env as IxonEnv; +use lean_ffi::object::{ + LeanArray, LeanBool, LeanBorrowed, LeanIOResult, LeanOwned, LeanString, +}; +use rustc_hash::{FxHashMap, FxHashSet}; + +/// FFI: pack a value bundle. +/// `(envPath mainName : String) → (assume : Array String) → +/// (outPath : String) → (verbose : Bool) → IO Unit`. +/// +/// `assume` entries resolve as displayed constant names first, else as +/// 64-hex constant addresses (cut points need not be named). +#[unsafe(no_mangle)] +pub extern "C" fn rs_pack_env( + env_path: LeanString>, + main_name: LeanString>, + assume: LeanArray>, + out_path: LeanString>, + verbose: LeanBool>, +) -> LeanIOResult { + let verbose = verbose.to_bool(); + let path = env_path.to_string(); + let main_str = main_name.to_string(); + let out = out_path.to_string(); + let assume_vec: Vec = assume.map(|obj| obj.as_string().to_string()); + + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to read {path}: {e}" + )); + }, + }; + if verbose { + eprintln!( + "[rs_pack_env] parsing {path} ({} MB, full reader)...", + bytes.len() / 1_000_000 + ); + } + let mut slice: &[u8] = &bytes; + let full = match IxonEnv::get(&mut slice) { + Ok(env) => env, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to deserialize {path}: {e}" + )); + }, + }; + if verbose { + eprintln!( + "[rs_pack_env] source env: {} consts, {} named, {} blobs", + full.consts.len(), + full.named.len(), + full.blobs.len() + ); + } + + // Resolve displayed names → addresses through the full env's `named` + // metadata (the `rs_env_extract` idiom). + let by_name: FxHashMap = full + .named + .iter() + .map(|e| (e.key().to_string(), e.value().addr.clone())) + .collect(); + let main = match by_name.get(&main_str) { + Some(a) => a.clone(), + None => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: no constant named {main_str} in {path}" + )); + }, + }; + let mut assumed: FxHashSet
= FxHashSet::default(); + let mut unresolved: Vec<&str> = Vec::new(); + for s in &assume_vec { + if let Some(a) = by_name.get(s.as_str()) { + assumed.insert(a.clone()); + } else if let Some(a) = Address::from_hex(s) { + assumed.insert(a); + } else { + unresolved.push(s); + } + } + if !unresolved.is_empty() { + return LeanIOResult::error_string(&format!( + "rs_pack_env: --assume entries neither named in {path} nor 64-hex \ + addresses: [{}]", + unresolved.join(", ") + )); + } + + let bundle = match full.prune_to_closure(&main, &assumed) { + Ok(b) => b, + Err(e) => return LeanIOResult::error_string(&format!("rs_pack_env: {e}")), + }; + if let Err(e) = bundle.validate_closed() { + return LeanIOResult::error_string(&format!("rs_pack_env: {e}")); + } + let mut buf = Vec::new(); + if let Err(e) = bundle.put(&mut buf) { + return LeanIOResult::error_string(&format!( + "rs_pack_env: bundle serialization failed: {e}" + )); + } + if let Err(e) = std::fs::write(&out, &buf) { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to write {out}: {e}" + )); + } + if verbose { + eprintln!("[rs_pack_env] main {} ({main_str})", main.hex()); + eprintln!( + "[rs_pack_env] kept {}/{} consts, {}/{} named, {}/{} blobs, \ + {} assumption(s)", + bundle.consts.len(), + full.consts.len(), + bundle.named.len(), + full.named.len(), + bundle.blobs.len(), + full.blobs.len(), + bundle.assumptions.len() + ); + eprintln!("[rs_pack_env] wrote {out} ({} bytes)", buf.len()); + } + LeanIOResult::ok(LeanOwned::box_usize(0)) +} diff --git a/docs/Ixon.md b/docs/Ixon.md index 70418060f..696bddfc8 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -930,6 +930,22 @@ behind those addresses. - Whole-environment files are the degenerate case (`main` absent, `assumptions` empty). +The CLI producer is `ix pack`: + +``` +lake exe ix pack [--out ] + [--assume ] [--assume-file ] [--verbose] +``` + +It resolves `` (displayed form) against the env's `named` table, +runs `prune_to_closure`, re-validates with `validate_closed`, and +writes the bundle (default `.ixe`). `--assume` entries — names or +64-hex constant addresses — declare trust-boundary cut points; the ones +actually reached become the bundle's `assumptions` (thin bundle). Pack +reads with the full reader so display metadata survives the prune; +compile once, pack many. (`ix shard extract` is the non-bundle sibling: +a general sub-env for the kernel-check pipeline, no `main` root.) + --- ## Proofs and Claims From a030bf58a54e1b029a63e85483642a7cfc02f907 Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Fri, 10 Jul 2026 00:52:30 -0400 Subject: [PATCH 5/8] Ixon: ix diff ripple root-causing (root vs rippled classification) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One edited constant re-addresses its whole reverse-dependency cone, so a content-addressed diff drowns the human edits in transitive re-addressings. Every changed row now carries a verdict: after the named join, pass 2 re-classifies each changed pair under a quotient where an (old, new) address pair compares equal when some changed name maps old→new. Rows whose residual labels are all "encoding" / "block-siblings" are rippled (fully explained by dependency re-addressing); the rest are roots. A single-level map is complete because expression comparison only consults immediate-dependency addresses — composition happens through per-row verdicts. Engine (ixon::diff): set-valued AddrMap built in the join loop (kind and encoding rows included; tolerates name splits); ExprCmp/classify parameterized by Option<&AddrMap> at the three address sites (Ref/Str+Nat/Prj-expr); classify_prj block short-circuit stays strict == and always descends (blocks are unnamed — a mapped shortcut would hide block-internal roots); pass-2 verdict cache keyed on the (old, new) pair so alias rows share one computation; constants re-parse rather than stash (LazyConstant no-cache policy keeps peak RSS flat). JoinProgress gains DiffPhase {NamedJoin, RippleClassify} with a 10k ripple stride. NamedChange.rippled crosses the FFI as a num_8 scalar (fresh scalar region; obj slots untouched). CLI: the counts line reports "(N roots, M rippled)"; the default listing shows roots only (plus, under --meta, rippled rows carrying metadata edits — namedMetaOnly only covers same-addr rows); --verbose lists rippled rows with a " (rippled)" suffix. Documented semantic edges: induced re-elaboration (a dependency universe/arity change alters dependents beyond addresses) verdicts root; an intrinsically edited block member with no named projection has no root row; the "block" fallback on assumption-cut bundles verdicts root — all fail-safe over-reports. Tests: 14 new engine tests (two-hop chain, block-ctor edit descent, name splits, kind-row mapping, univ-arity root, renamed dep, literal root, encoding ripple, alias cache, Prj-expr site, cidx root, …); join-progress test now asserts per-phase; Lean FFI marshaling tests pin the num_8 slot. Mathlib validation (v4.29.0 ↔ 8850ed93, 744k joined names): 143,528 changed → 4,516 roots / 139,012 rippled (96.9% explained). All 5 kind changes, all 245 lvls signature changes, and 8 genuine ctor-type edits surface as roots; 605 induced block.ctor.type rows correctly ripple; zero rippled rows carry scalar labels; deterministic across runs. Ripple pass +16.5s (compute 30.8s vs 14.3s), peak RSS unchanged at 54 GiB. --- Ix/Cli/DiffCmd.lean | 36 +- Ix/Ixon.lean | 8 +- Tests/FFI/Ixon.lean | 26 +- crates/ffi/src/lean.rs | 3 +- crates/ffi/src/lean_ixon/diff.rs | 19 +- crates/ixon/src/diff.rs | 745 +++++++++++++++++++++++++++++-- crates/ixon/src/lib.rs | 3 +- 7 files changed, 778 insertions(+), 62 deletions(-) diff --git a/Ix/Cli/DiffCmd.lean b/Ix/Cli/DiffCmd.lean index 888ae1bd1..799ea027f 100644 --- a/Ix/Cli/DiffCmd.lean +++ b/Ix/Cli/DiffCmd.lean @@ -10,6 +10,14 @@ representation moved). `--meta` additionally compares named metadata (`ConstantMeta`/`original`). + Every changed row carries a root-vs-rippled verdict: one edited + constant re-addresses its whole reverse-dependency cone, so most + changed rows are *rippled* (fully explained by dependency + re-addressing) and only the *roots* are intrinsic edits. The default + display lists roots and summarizes the rippled count; `--verbose` + lists rippled rows too, and in `--meta` mode rippled rows carrying + metadata edits stay visible. + Exit codes (GNU diff convention): 0 = no difference found in the selected mode, 1 = differences found, 2 = error. -/ @@ -62,35 +70,49 @@ private def printNamedSection let removed := d.namedRemoved.filter (keep ·.1) let synAdded := d.namedAdded.size - added.size let synRemoved := d.namedRemoved.size - removed.size + let roots := d.namedChanged.filter (!·.rippled) + let rippleCount := + if d.namedChanged.isEmpty then "" + else s!" ({roots.size} roots, {d.namedChanged.size - roots.size} rippled)" let metaCount := if wantMeta then s!", {d.namedMetaOnly.size} metadata-only" else "" IO.println - s!"named: {d.namedAdded.size} added, {d.namedRemoved.size} removed, {d.namedChanged.size} changed{metaCount}" + s!"named: {d.namedAdded.size} added, {d.namedRemoved.size} removed, {d.namedChanged.size} changed{rippleCount}{metaCount}" if synAdded + synRemoved > 0 then IO.println s!" (synthetic mutual-block names: {synAdded} added, {synRemoved} removed — --verbose lists)" + -- Changed rows shown by default: the roots, plus (under --meta) + -- rippled rows carrying metadata edits — `namedMetaOnly` only covers + -- same-addr rows, so hiding those would hide real metadata changes. + let shown := d.namedChanged.filter fun c => + verbose || !c.rippled || (wantMeta && !c.metaFields.isEmpty) -- Column width over everything we are about to print. let mut wMax := 0 for (n, _) in added do wMax := max wMax n.length for (n, _) in removed do wMax := max wMax n.length - for c in d.namedChanged do wMax := max wMax c.name.length + for c in shown do wMax := max wMax c.name.length for (n, _) in d.namedMetaOnly do wMax := max wMax n.length let w := min wMax 40 for (n, addr) in added do IO.println s!" + {pad n w} {shortAddr verbose addr}" for (n, addr) in removed do IO.println s!" - {pad n w} {shortAddr verbose addr}" - for c in d.namedChanged do + for c in shown do let kind := if c.oldKind == c.newKind then c.oldKind else s!"{c.oldKind}→{c.newKind}" + let ripTag := if c.rippled then " (rippled)" else "" IO.println - s!" ~ {pad c.name w} {pad kind 9} {shortAddr verbose c.oldAddr} → {shortAddr verbose c.newAddr} {brackets c.fields}" + s!" ~ {pad c.name w} {pad kind 9} {shortAddr verbose c.oldAddr} → {shortAddr verbose c.newAddr} {brackets c.fields}{ripTag}" if wantMeta && !c.metaFields.isEmpty then IO.println s!" meta: {brackets c.metaFields}" + let hidden := d.namedChanged.size - shown.size + if hidden > 0 then + IO.println + s!" ({hidden} rippled rows hidden — address changes fully explained by dependency re-addressing; --verbose lists)" for (n, labels) in d.namedMetaOnly do IO.println s!" m {pad n w} {brackets labels}" - if d.namedChanged.any (·.fields.contains "encoding") then + if shown.any (·.fields.contains "encoding") then IO.println " (encoding = representation changed; no semantic field difference detected)" @@ -175,12 +197,12 @@ end Ix.Cli.DiffCmd open Ix.Cli.DiffCmd in def diffCmd : Cli.Cmd := `[Cli| diff VIA runDiffCmd; - "Print a structured diff of two serialized Ixon environments (`.ixe`). By default only anonymous structure is compared: constants by content address (joined through names, with per-field change classification), consts/blobs sets, comms, main/assumptions, and reducibility hints. Exit codes: 0 = no difference, 1 = differences found, 2 = error." + "Print a structured diff of two serialized Ixon environments (`.ixe`). By default only anonymous structure is compared: constants by content address (joined through names, with per-field change classification), consts/blobs sets, comms, main/assumptions, and reducibility hints. Changed names are root-caused: rows fully explained by dependency re-addressing are counted as `rippled` and hidden by default, so the listing shows the intrinsic edits (roots). Exit codes: 0 = no difference, 1 = differences found, 2 = error." FLAGS: anon; "Compare only anonymous structure (the default; accepted for explicitness)." «meta»; "Additionally compare named metadata (binder names, originals, kv-maps)." - verbose; "Print full addresses, uncapped lists, and synthetic mutual-block names." + verbose; "Print full addresses, uncapped lists, synthetic mutual-block names, and rippled changed rows." ARGS: old : String; "Path to the first (old) serialized env (`.ixe`)." diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index b48b40b32..07c9eec3e 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -2294,7 +2294,12 @@ structure EnvStats where `fields` is never empty: `"kind"` marks a variant change and `"encoding"` an address change with no detected semantic field difference (table reorder / sharing-decision churn). `metaFields` - is only populated in meta mode. -/ + is only populated in meta mode. `rippled` is the root-cause + verdict: true iff the address change is fully explained by + dependency re-addressing (re-classified under the old→new quotient + of all changed rows, every residual label is + `"encoding"`/`"block-siblings"`); `fields` stays the strict + classification. -/ structure NamedDiff where name : String oldAddr : Address @@ -2303,6 +2308,7 @@ structure NamedDiff where newKind : String fields : Array String metaFields : Array String + rippled : Bool deriving Inhabited, BEq /-- Report produced by `rsDiffEnvs`. Set-difference lists are complete diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index 44119ecb7..0c855a483 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -331,7 +331,31 @@ def envDiffTests : TestSeq := (d.namedChanged[0]!).oldKind == "defn" && (d.namedChanged[0]!).newKind == "defn" && (d.namedChanged[0]!).fields == #["value"] && - (d.namedChanged[0]!).metaFields.isEmpty) ++ + (d.namedChanged[0]!).metaFields.isEmpty && + (d.namedChanged[0]!).rippled == false) ++ + -- A dependent whose only change is its ref re-addressing must marshal + -- `rippled == true` (pins the num_8 scalar slot against layout swaps). + test "EnvDiff: ripple verdict marshals" + (let mkRefConst (r : Address) : Constant := + { info := .defn { kind := .defn, safety := .safe, lvls := 0, + typ := .var 3, value := .ref 0 #[] } + sharing := #[], refs := #[r], univs := #[] } + let mkPair (leaf : Constant) : Env := Id.run do + let leafAddr := Address.blake3 (serConstant leaf) + let dep := mkRefConst leafAddr + let depAddr := Address.blake3 (serConstant dep) + let mut env : Env := {} + env := env.storeConst leafAddr leaf + env := env.storeConst depAddr dep + env := { env with names := RawEnv.addNameComponents env.names fooName } + env := env.registerName fooName { addr := leafAddr, constMeta := .empty } + env := { env with names := RawEnv.addNameComponents env.names barName } + env := env.registerName barName { addr := depAddr, constMeta := .empty } + return env + (runDiff (mkPair constA) (mkPair constB)).any fun d => + d.namedChanged.size == 2 && + (d.namedChanged.find? (·.name == "foo")).any (·.rippled == false) && + (d.namedChanged.find? (·.name == "bar")).any (·.rippled == true)) ++ -- Hints derive into anon §3, so a hint tweak is visible in the -- default anon mode; the metadata carrying it only shows under meta. test "EnvDiff: hint change visible in anon mode" diff --git a/crates/ffi/src/lean.rs b/crates/ffi/src/lean.rs index f3fb24afa..dab1d3e74 100644 --- a/crates/ffi/src/lean.rs +++ b/crates/ffi/src/lean.rs @@ -67,7 +67,8 @@ lean_ffi::lean_inductive! { // consts, named, blobs, comms counts LeanIxonEnvStats [ { num_64: 4 } ]; // name, oldAddr, newAddr, oldKind, newKind, fields, metaFields - LeanIxonNamedDiff [ { num_obj: 7 } ]; + // + rippled (Bool scalar) + LeanIxonNamedDiff [ { num_obj: 7, num_8: 1 } ]; // mainChanged, assumptions±, named±/changed/metaOnly, comms±/changed, // constsOnly×2, blobsOnly×2, hintsChanged, statsA, statsB LeanIxonEnvDiff [ { num_obj: 17 } ]; diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs index 8f0b7f875..c5ac2530e 100644 --- a/crates/ffi/src/lean_ixon/diff.rs +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -14,7 +14,7 @@ use std::time::Instant; use ix_common::address::Address; -use ixon::diff::{EnvDiff, EnvStats, NamedChange, diff_envs_with}; +use ixon::diff::{DiffPhase, EnvDiff, EnvStats, NamedChange, diff_envs_with}; use ixon::env::Env as IxonEnv; use lean_ffi::object::{ LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, @@ -95,7 +95,7 @@ impl LeanIxonEnvStats { impl LeanIxonNamedDiff { /// Build `Ixon.NamedDiff { name, oldAddr, newAddr, oldKind, newKind, - /// fields, metaFields }`. + /// fields, metaFields, rippled }`. fn build(c: &NamedChange) -> Self { let ctor = LeanIxonNamedDiff::alloc(0); ctor.set_obj(0, LeanString::new(&c.name)); @@ -105,6 +105,7 @@ impl LeanIxonNamedDiff { ctor.set_obj(4, LeanString::new(c.new_kind)); ctor.set_obj(5, string_array(&c.fields)); ctor.set_obj(6, string_array(&c.meta_fields)); + ctor.set_num_8(0, u8::from(c.rippled)); ctor } } @@ -208,10 +209,16 @@ pub extern "C" fn rs_diff_envs( let t = Instant::now(); let mut on_progress = |p: ixon::diff::JoinProgress| { if progress { - eprintln!( - "[rs_diff_envs] named join: {}/{} ({} changed so far)", - p.done, p.total, p.changed - ); + match p.phase { + DiffPhase::NamedJoin => eprintln!( + "[rs_diff_envs] named join: {}/{} ({} changed so far)", + p.done, p.total, p.changed + ), + DiffPhase::RippleClassify => eprintln!( + "[rs_diff_envs] ripple pass: {}/{} ({} roots so far)", + p.done, p.total, p.changed + ), + } } }; match diff_envs_with(&env_a, &env_b, want_meta, &mut on_progress) { diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs index 5307ffa58..60c2a22ef 100644 --- a/crates/ixon/src/diff.rs +++ b/crates/ixon/src/diff.rs @@ -19,6 +19,28 @@ //! `Named.original` content, populating [`EnvDiff::named_meta_only`] //! and [`NamedChange::meta_fields`]. //! +//! Every changed row additionally carries a ripple verdict +//! ([`NamedChange::rippled`]): after the join, changed pairs are +//! re-classified under a quotient where an (old, new) address pair +//! compares equal when some changed name maps old→new. Rows whose +//! residual labels are all `"encoding"`/`"block-siblings"` are +//! *rippled* — fully explained by dependency re-addressing (the +//! content-address ripple: one edited constant re-addresses its whole +//! reverse-dependency cone); the rest are *roots* (intrinsic edits). A +//! single-level mapping is complete because expression comparison only +//! ever consults immediate-dependency addresses — composition across +//! the DAG happens through the per-row verdicts, never through the map. +//! Accepted semantic edges: +//! - Induced re-elaboration verdicts root: a dependency's +//! universe/arity signature change alters dependents' terms beyond +//! addresses (e.g. `Ref` univ-argument arity), so "roots" +//! over-approximates "human edits" when signatures change. +//! - An intrinsic edit of a block member with no named projection has +//! no root row of its own (blocks are unnamed; in practice ix names +//! every projection, so the member's projection row is the root). +//! - The `"block"` fallback (block bytes missing, e.g. behind a +//! bundle's assumption cut) verdicts root — fail-safe over-report. +//! //! Expression comparison ([`exprs_equal`]) is a lockstep structural walk //! that resolves table indices through each side's own `Constant` tables: //! identical expr bytes over different tables compare *different*, and @@ -32,7 +54,9 @@ //! prefixed) churn as one removed+added pair per changed block; display //! layers may group them. -use rustc_hash::FxHashSet; +use std::sync::Arc; + +use rustc_hash::{FxHashMap, FxHashSet}; use ix_common::address::Address; use ix_common::env::{Name, ReducibilityHints}; @@ -72,6 +96,12 @@ pub struct NamedChange { /// Metadata component labels when the metadata ALSO changed. Only /// populated in meta mode; may be empty. pub meta_fields: Vec, + /// True iff the address change is fully explained by dependency + /// re-addressing: re-classified under the old→new quotient of all + /// changed rows, the residual labels are all `"encoding"` / + /// `"block-siblings"`. `fields` stays the strict (unquotiented) + /// classification; this is the orthogonal root-vs-ripple verdict. + pub rippled: bool, } /// Report produced by [`diff_envs`]. All name-keyed vectors are sorted @@ -205,6 +235,12 @@ fn ref_at<'c>( }) } +/// Old→new address pairs harvested from every changed named row in +/// pass 1 (kind-change and encoding rows included). Set-valued to +/// tolerate name splits: two names sharing an old address, diverging in +/// the new env. Directional — keys are A-side (old) addresses. +type AddrMap = FxHashMap>; + /// Lockstep expression comparator over one pair of table contexts. /// /// The pointer-pair memo persists across `eq` calls for the same @@ -212,15 +248,27 @@ fn ref_at<'c>( /// across type/value/rules). Entries record "heads matched, children /// enqueued" — an unequal result returns early and leaves enqueued /// children unverified, so [`Self::eq`] clears the memo on `false`. +/// +/// `map` selects the address equality: `None` = strict (pass 1), +/// `Some` = the ripple quotient (pass 2). struct ExprCmp<'a> { ca: &'a Constant, cb: &'a Constant, memo: FxHashSet<(usize, usize)>, + map: Option<&'a AddrMap>, } impl<'a> ExprCmp<'a> { - fn new(ca: &'a Constant, cb: &'a Constant) -> Self { - ExprCmp { ca, cb, memo: FxHashSet::default() } + fn new(ca: &'a Constant, cb: &'a Constant, map: Option<&'a AddrMap>) -> Self { + ExprCmp { ca, cb, memo: FxHashSet::default(), map } + } + + /// Address equality under the selected mode. `l` MUST be the A-side + /// (old) address and `r` the B-side (new) — the quotient map is + /// directional. Every comparison site preserves that order (the walk + /// always pairs (A-expr, B-expr)). + fn addrs_eq(&self, l: &Address, r: &Address) -> bool { + l == r || self.map.is_some_and(|m| m.get(l).is_some_and(|v| v.contains(r))) } fn eq(&mut self, a: &'a Expr, b: &'a Expr) -> Result { @@ -254,8 +302,10 @@ impl<'a> ExprCmp<'a> { } }, (Expr::Ref(i, us), Expr::Ref(j, vs)) => { - if ref_at(self.ca, *i, "left")? != ref_at(self.cb, *j, "right")? - || !self.univ_lists_equal(us, vs)? + if !self.addrs_eq( + ref_at(self.ca, *i, "left")?, + ref_at(self.cb, *j, "right")?, + ) || !self.univ_lists_equal(us, vs)? { return Ok(false); } @@ -269,13 +319,22 @@ impl<'a> ExprCmp<'a> { (Expr::Str(i), Expr::Str(j)) | (Expr::Nat(i), Expr::Nat(j)) => { // Blob content behind equal addresses is identical by // content-addressing (blob sections are hash-verified on load). - if ref_at(self.ca, *i, "left")? != ref_at(self.cb, *j, "right")? { + // Blob addresses never enter the quotient map (it holds + // constant addresses from named rows), so literal changes stay + // intrinsic under pass 2 — `addrs_eq` degenerates to `==`. + if !self.addrs_eq( + ref_at(self.ca, *i, "left")?, + ref_at(self.cb, *j, "right")?, + ) { return Ok(false); } }, (Expr::Prj(ti, fi, va), Expr::Prj(tj, fj, vb)) => { if fi != fj - || ref_at(self.ca, *ti, "left")? != ref_at(self.cb, *tj, "right")? + || !self.addrs_eq( + ref_at(self.ca, *ti, "left")?, + ref_at(self.cb, *tj, "right")?, + ) { return Ok(false); } @@ -326,7 +385,7 @@ pub fn exprs_equal( b: &Expr, cb: &Constant, ) -> Result { - ExprCmp::new(ca, cb).eq(a, b) + ExprCmp::new(ca, cb, None).eq(a, b) } // ============================================================================ @@ -518,6 +577,7 @@ fn classify_prj( cidx: Option<(u64, u64)>, block_a: &Address, block_b: &Address, + map: Option<&AddrMap>, out: &mut Vec, ) -> Result<(), String> { if idx_a != idx_b { @@ -528,6 +588,11 @@ fn classify_prj( { out.push("cidx".to_string()); } + // Strict `==` deliberately, even under the quotient: blocks are + // unnamed (their synthetic names embed the hash, so they never join + // as changed rows and never enter the map), and descent is mandatory + // — a block-internal intrinsic edit has no named row of its own, so + // short-circuiting a "mapped" block pair would hide the root. if block_a == block_b { return Ok(()); } @@ -558,7 +623,7 @@ fn classify_prj( }; // Expression tables are constant-level: members resolve through the // enclosing block constants' tables. - let mut cmp = ExprCmp::new(&ba, &bb); + let mut cmp = ExprCmp::new(&ba, &bb, map); let before = out.len(); match cidx { None => classify_mut_member(mem_a, mem_b, &mut cmp, "block.", out)?, @@ -589,14 +654,19 @@ fn classify_prj( /// labels. Never returns an empty list: kind mismatches yield `"kind"`, /// and an address change with no detected semantic difference yields /// `"encoding"` (table reorder / sharing-decision churn). +/// +/// `map = None` is the strict pass-1 classification; `Some` re-runs it +/// under the ripple quotient (pass 2), where the labels are consumed +/// only by the [`rippled_labels`] verdict and then discarded. fn classify_constants( env_a: &Env, ca: &Constant, env_b: &Env, cb: &Constant, + map: Option<&AddrMap>, ) -> Result, String> { let mut out = Vec::new(); - let mut cmp = ExprCmp::new(ca, cb); + let mut cmp = ExprCmp::new(ca, cb, map); match (&ca.info, &cb.info) { (ConstantInfo::Defn(a), ConstantInfo::Defn(b)) => { classify_defn(a, b, &mut cmp, "", &mut out)?; @@ -628,17 +698,17 @@ fn classify_constants( }, (ConstantInfo::DPrj(a), ConstantInfo::DPrj(b)) => { classify_prj( - env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, )?; }, (ConstantInfo::IPrj(a), ConstantInfo::IPrj(b)) => { classify_prj( - env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, )?; }, (ConstantInfo::RPrj(a), ConstantInfo::RPrj(b)) => { classify_prj( - env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, &mut out, + env_a, env_b, a.idx, b.idx, None, &a.block, &b.block, map, &mut out, )?; }, (ConstantInfo::CPrj(a), ConstantInfo::CPrj(b)) => { @@ -650,6 +720,7 @@ fn classify_constants( Some((a.cidx, b.cidx)), &a.block, &b.block, + map, &mut out, )?; }, @@ -727,6 +798,31 @@ fn env_stats(e: &Env) -> EnvStats { } } +/// Materialize the constant behind a named row's address, with +/// name-and-side error context. Shared by the pass-1 classification and +/// the pass-2 ripple verdict (constants are re-parsed on every call — +/// `LazyConstant` deliberately keeps no parse cache, which is what +/// bounds resident memory at mathlib scale). +fn materialize_const( + env: &Env, + addr: &Address, + name: &Name, + which: &str, +) -> Result, String> { + env + .try_get_const(addr) + .ok_or_else(|| { + format!( + "diff: named '{}' addr {} not present in consts of the {which} env", + name.pretty(), + addr.hex() + ) + })? + .map_err(|e| { + format!("diff: named '{}' ({}): {e}", name.pretty(), addr.hex()) + }) +} + fn classify_named_change( env_a: &Env, env_b: &Env, @@ -735,23 +831,9 @@ fn classify_named_change( nb: &Named, meta: bool, ) -> Result { - let materialize = |env: &Env, addr: &Address, which: &str| { - env - .try_get_const(addr) - .ok_or_else(|| { - format!( - "diff: named '{}' addr {} not present in consts of the {which} env", - name.pretty(), - addr.hex() - ) - })? - .map_err(|e| { - format!("diff: named '{}' ({}): {e}", name.pretty(), addr.hex()) - }) - }; - let ca = materialize(env_a, &na.addr, "first")?; - let cb = materialize(env_b, &nb.addr, "second")?; - let fields = classify_constants(env_a, &ca, env_b, &cb)?; + let ca = materialize_const(env_a, &na.addr, name, "first")?; + let cb = materialize_const(env_b, &nb.addr, name, "second")?; + let fields = classify_constants(env_a, &ca, env_b, &cb, None)?; let meta_fields = if meta { meta_component_labels(na, nb) } else { Vec::new() }; Ok(NamedChange { @@ -762,32 +844,66 @@ fn classify_named_change( new_kind: kind_label(&cb.info), fields, meta_fields, + // Verdict assigned by pass 2 (a row is a root until explained). + rippled: false, }) } +/// Pass-2 verdict over quotient labels: rippled iff every residual +/// label is explained by re-addressing — `"encoding"` (quotient-equal +/// everywhere; the fallback when no field label fired) or +/// `"block-siblings"` (the projected member is quotient-equal; the +/// block hash moved for reasons surfaced by other rows). Scalar labels, +/// `"kind"`, coordinate labels (`"idx"`/`"cidx"`), `"block"`, and all +/// `block.*` detail labels mean intrinsic change → root. Labels are +/// never empty (the `"encoding"` fallback), so `all` cannot pass +/// vacuously. +fn rippled_labels(labels: &[String]) -> bool { + labels.iter().all(|l| l == "encoding" || l == "block-siblings") +} + /// Sort by (pretty name, name hash) — the hash tiebreak keeps distinct /// names with equal pretty forms deterministic. fn sort_by_name(v: &mut [(String, Name, T)]) { v.sort_by(|x, y| x.0.cmp(&y.0).then_with(|| x.1.cmp(&y.1))); } -/// Coarse progress for the named join — the only diff phase whose cost -/// scales with the number of *changed* names (each one materializes two -/// constants and walks their expressions). Parses happen before -/// [`diff_envs`] and are the caller's to report. +/// Which diff pass a [`JoinProgress`] event reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DiffPhase { + /// Pass 1: the named join (change detection + strict classification). + NamedJoin, + /// Pass 2: re-classification of changed rows under the ripple + /// quotient (root vs rippled verdicts). + RippleClassify, +} + +/// Coarse progress for the two passes whose cost scales with the number +/// of *changed* names (each one materializes two constants and walks +/// their expressions). Parses happen before [`diff_envs`] and are the +/// caller's to report. #[derive(Clone, Copy, Debug)] pub struct JoinProgress { - /// Entries of the first env's named map processed so far. + /// The pass this event reports. `done` counts reset between phases. + pub phase: DiffPhase, + /// [`DiffPhase::NamedJoin`]: first-env named entries processed. + /// [`DiffPhase::RippleClassify`]: changed rows verdicted. pub done: usize, - /// Total entries in the first env's named map. + /// Total entries of the reporting phase. pub total: usize, - /// Changed names (addr differs) classified so far. + /// [`DiffPhase::NamedJoin`]: changed names classified so far. + /// [`DiffPhase::RippleClassify`]: roots found so far. pub changed: usize, } /// Named-join entries between progress callbacks in [`diff_envs_with`]. const JOIN_PROGRESS_STRIDE: usize = 100_000; +/// Changed rows between pass-2 progress callbacks — one event every +/// ~1-2 s at mathlib scale (each row re-materializes and re-walks a +/// constant pair, so rows are ~10× slower than join probes). +const RIPPLE_PROGRESS_STRIDE: usize = 10_000; + /// Diff two environments. `meta = false` (anon mode, the default) /// compares only anonymous structure; `meta = true` additionally /// compares `Named` metadata content. See the module docs. @@ -817,6 +933,8 @@ pub fn diff_envs_with( let mut removed: Vec<(String, Name, Address)> = Vec::new(); let mut changed: Vec<(String, Name, NamedChange)> = Vec::new(); let mut meta_only: Vec<(String, Name, Vec)> = Vec::new(); + // Old→new mapping over every changed row, consumed by pass 2. + let mut addr_map: AddrMap = FxHashMap::default(); let join_total = a.named.len(); let mut join_done: usize = 0; for entry in a.named.iter() { @@ -834,6 +952,10 @@ pub fn diff_envs_with( } } else { let change = classify_named_change(a, b, name, na, nb, meta)?; + let slot = addr_map.entry(na.addr.clone()).or_default(); + if !slot.contains(&nb.addr) { + slot.push(nb.addr.clone()); + } changed.push((name.pretty(), name.clone(), change)); } }, @@ -842,6 +964,7 @@ pub fn diff_envs_with( if join_done.is_multiple_of(JOIN_PROGRESS_STRIDE) && join_done < join_total { on_progress(JoinProgress { + phase: DiffPhase::NamedJoin, done: join_done, total: join_total, changed: changed.len(), @@ -849,10 +972,58 @@ pub fn diff_envs_with( } } on_progress(JoinProgress { + phase: DiffPhase::NamedJoin, done: join_total, total: join_total, changed: changed.len(), }); + + // Pass 2 (ripple): re-classify each changed pair under the quotient + // of the just-completed old→new mapping. A pair whose residual labels + // are all explained by re-addressing is rippled (induced by its + // dependencies' address changes); the rest are roots. Alias rows — + // several names spanning one (old, new) pair — share one cached + // verdict and one computation. Materialization failure here is a hard + // error: pass 1 already materialized both sides of every changed row. + if !changed.is_empty() { + let mut verdicts: FxHashMap<(Address, Address), bool> = + FxHashMap::default(); + let ripple_total = changed.len(); + let mut roots: usize = 0; + for (i, (_, name, ch)) in changed.iter_mut().enumerate() { + let key = (ch.old_addr.clone(), ch.new_addr.clone()); + let rippled = match verdicts.get(&key) { + Some(v) => *v, + None => { + let ca = materialize_const(a, &ch.old_addr, name, "first")?; + let cb = materialize_const(b, &ch.new_addr, name, "second")?; + let labels = classify_constants(a, &ca, b, &cb, Some(&addr_map))?; + let v = rippled_labels(&labels); + verdicts.insert(key, v); + v + }, + }; + ch.rippled = rippled; + if !rippled { + roots += 1; + } + let done = i + 1; + if done.is_multiple_of(RIPPLE_PROGRESS_STRIDE) && done < ripple_total { + on_progress(JoinProgress { + phase: DiffPhase::RippleClassify, + done, + total: ripple_total, + changed: roots, + }); + } + } + on_progress(JoinProgress { + phase: DiffPhase::RippleClassify, + done: ripple_total, + total: ripple_total, + changed: roots, + }); + } for entry in b.named.iter() { let name = entry.key(); if a.named.get(name).is_none() { @@ -966,7 +1137,9 @@ pub fn diff_envs_with( mod tests { use super::*; use crate::comm::Comm; - use crate::constant::{DefKind, defn_proj_constant}; + use crate::constant::{ + DefKind, ctor_proj_constant, defn_proj_constant, indc_proj_constant, + }; use crate::metadata::{ConstantMeta, ConstantMetaInfo, ExprMeta}; use ix_common::env::DefinitionSafety; use std::sync::Arc; @@ -1463,15 +1636,39 @@ mod tests { let mut events: Vec = Vec::new(); let d = diff_envs_with(&a, &b, false, &mut |p| events.push(p)) .expect("diff_envs_with failed"); - let last = events.last().expect("no progress events"); - assert_eq!(last.done, last.total); - assert_eq!(last.total, a.named.len()); - assert_eq!(last.changed, d.named_changed.len()); + // `done` resets between phases — assert each phase separately. + let join: Vec<_> = + events.iter().filter(|p| p.phase == DiffPhase::NamedJoin).collect(); + let ripple: Vec<_> = + events.iter().filter(|p| p.phase == DiffPhase::RippleClassify).collect(); + let last_join = join.last().expect("no join events"); + assert_eq!(last_join.done, last_join.total); + assert_eq!(last_join.total, a.named.len()); + assert_eq!(last_join.changed, d.named_changed.len()); assert_eq!(d.named_changed.len(), 1); - assert!( - events.windows(2).all(|w| w[0].done <= w[1].done), - "progress must be monotone" + let last_ripple = ripple.last().expect("no ripple events"); + assert_eq!(last_ripple.done, last_ripple.total); + assert_eq!(last_ripple.total, d.named_changed.len()); + assert_eq!( + last_ripple.changed, + d.named_changed.iter().filter(|c| !c.rippled).count() ); + for phase_events in [&join, &ripple] { + assert!( + phase_events.windows(2).all(|w| w[0].done <= w[1].done), + "per-phase progress must be monotone" + ); + } + // All ripple events come after the join completes. + let first_ripple = events + .iter() + .position(|p| p.phase == DiffPhase::RippleClassify) + .expect("no ripple events"); + let last_join_pos = events + .iter() + .rposition(|p| p.phase == DiffPhase::NamedJoin) + .expect("no join events"); + assert!(last_join_pos < first_ripple, "phases must not interleave"); } #[test] @@ -1504,4 +1701,462 @@ mod tests { let d = diff(&a, &b); assert_eq!(labels(&d.named_changed[0].fields), ["rules.len"]); } + + // ========================================================================== + // Ripple root-causing (pass 2) + // ========================================================================== + + /// A defn whose value is a bare `Ref` into `refs[0] = r`. + fn defn_ref(typ_var: u64, r: Address) -> Constant { + defn_ct( + Expr::var(typ_var), + Expr::reference(0, vec![]), + vec![], + vec![r], + vec![], + ) + } + + fn changed_by_name<'d>(d: &'d EnvDiff, name: &str) -> &'d NamedChange { + d.named_changed + .iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("no changed row named {name}: {d:?}")) + } + + /// Editing a leaf re-addresses the chain above it; only the leaf is a + /// root. Proves the single-level map composes across the DAG through + /// per-row verdicts. + #[test] + fn ripple_two_hop_chain() { + let build = |leaf_value: Arc| { + let env = Env::new(); + let leaf = defn_c(Expr::var(3), leaf_value); + let leaf_addr = store_canonical(&env, &leaf); + let mid = defn_ref(4, leaf_addr.clone()); + let mid_addr = store_canonical(&env, &mid); + let top = defn_ref(5, mid_addr.clone()); + let top_addr = store_canonical(&env, &top); + env.register_name(n("Leaf"), Named::with_addr(leaf_addr)); + env.register_name(n("Mid"), Named::with_addr(mid_addr)); + env.register_name(n("Top"), Named::with_addr(top_addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 3); + let leaf = changed_by_name(&d, "Leaf"); + assert!(!leaf.rippled, "the edited leaf is the root"); + assert_eq!(labels(&leaf.fields), ["value"]); + for name in ["Mid", "Top"] { + let c = changed_by_name(&d, name); + assert!(c.rippled, "{name} must be rippled: {c:?}"); + // Strict fields are untouched by the verdict. + assert_eq!(labels(&c.fields), ["value"]); + } + } + + /// Blob addresses never enter the quotient map: a literal edit is the + /// intrinsic change site. + #[test] + fn ripple_literal_change_is_root() { + let build = |blob: Vec| { + let env = Env::new(); + let blob_addr = env.store_blob(blob); + let c = + defn_ct(Expr::var(3), Expr::nat(0), vec![], vec![blob_addr], vec![]); + let addr = store_canonical(&env, &c); + env.register_name(n("Lit"), Named::with_addr(addr)); + env + }; + let a = build(vec![42]); + let b = build(vec![43]); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Lit"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["value"]); + } + + /// Block-internal ctor edit: the edited ctor's projection is the root + /// (block descent runs under the quotient too); the untouched sibling + /// ctor and any external dependent are rippled. + #[test] + fn ripple_block_ctor_edit_projections() { + let build = |c1_typ: Arc| { + let env = Env::new(); + let indc = Inductive { + is_unsafe: false, + lvls: 0, + params: 0, + indices: 0, + typ: Expr::var(3), + ctors: vec![ + Constructor { + is_unsafe: false, + lvls: 0, + cidx: 0, + params: 0, + fields: 0, + typ: Expr::var(1), + }, + Constructor { + is_unsafe: false, + lvls: 0, + cidx: 1, + params: 0, + fields: 0, + typ: c1_typ, + }, + ], + }; + let block = Constant::new(ConstantInfo::Muts(vec![MutConst::Indc(indc)])); + let block_addr = store_canonical(&env, &block); + let iprj = + store_canonical(&env, &indc_proj_constant(0, block_addr.clone())); + let c0 = + store_canonical(&env, &ctor_proj_constant(0, 0, block_addr.clone())); + let c1 = store_canonical(&env, &ctor_proj_constant(0, 1, block_addr)); + env.register_name(n("I"), Named::with_addr(iprj.clone())); + env.register_name(n("I.c0"), Named::with_addr(c0)); + env.register_name(n("I.c1"), Named::with_addr(c1)); + // External dependent referencing the inductive's projection. + let dep = defn_ref(9, iprj); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("UsesI"), Named::with_addr(dep_addr)); + env + }; + let a = build(Expr::var(2)); + let b = build(Expr::var(9)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 4); + + let edited = changed_by_name(&d, "I.c1"); + assert!(!edited.rippled, "edited ctor's projection is the root"); + assert_eq!(labels(&edited.fields), ["block.ctor.type"]); + + let indc_row = changed_by_name(&d, "I"); + assert!(!indc_row.rippled, "the inductive sees its ctor's edit"); + assert_eq!(labels(&indc_row.fields), ["block.ctors[1].type"]); + + let sibling = changed_by_name(&d, "I.c0"); + assert!(sibling.rippled, "untouched sibling ctor is rippled"); + assert_eq!(labels(&sibling.fields), ["block-siblings"]); + + let dep = changed_by_name(&d, "UsesI"); + assert!(dep.rippled, "external dependent of the block is rippled"); + assert_eq!(labels(&dep.fields), ["value"]); + } + + /// The existing block-descent fixture, now with verdicts: the member + /// whose value changed is the root; its sibling projection rides. + #[test] + fn ripple_defn_block_sibling() { + let f = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(0))); + let g_old = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(1))); + let g_new = MutConst::Defn(mk_defn(0, Expr::var(3), Expr::var(2))); + let block_a = Constant::new(ConstantInfo::Muts(vec![f.clone(), g_old])); + let block_b = Constant::new(ConstantInfo::Muts(vec![f, g_new])); + + let a = Env::new(); + let block_a_addr = store_canonical(&a, &block_a); + let fa = store_canonical(&a, &defn_proj_constant(0, block_a_addr.clone())); + let ga = store_canonical(&a, &defn_proj_constant(1, block_a_addr)); + a.register_name(n("M.f"), Named::with_addr(fa)); + a.register_name(n("M.g"), Named::with_addr(ga)); + + let b = Env::new(); + let block_b_addr = store_canonical(&b, &block_b); + let fb = store_canonical(&b, &defn_proj_constant(0, block_b_addr.clone())); + let gb = store_canonical(&b, &defn_proj_constant(1, block_b_addr)); + b.register_name(n("M.f"), Named::with_addr(fb)); + b.register_name(n("M.g"), Named::with_addr(gb)); + + let d = diff(&a, &b); + assert!(changed_by_name(&d, "M.f").rippled); + assert!(!changed_by_name(&d, "M.g").rippled); + } + + /// Scalar changes are quotient-invariant → always roots. + #[test] + fn ripple_lvls_change_is_root() { + let (a, _) = env1( + "Foo", + &Constant::new(ConstantInfo::Defn(mk_defn( + 0, + Expr::var(3), + Expr::var(0), + ))), + ); + let (b, _) = env1( + "Foo", + &Constant::new(ConstantInfo::Defn(mk_defn( + 1, + Expr::var(3), + Expr::var(0), + ))), + ); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["lvls"]); + } + + /// Pure representation churn (strict `["encoding"]`) has no intrinsic + /// difference under the quotient either → rippled. + #[test] + fn ripple_pure_encoding_is_rippled() { + let x = Address::hash(b"X"); + let y = Address::hash(b"Y"); + let ca = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(0, vec![]), Expr::reference(1, vec![])), + vec![], + vec![x.clone(), y.clone()], + vec![], + ); + let cb = defn_ct( + Expr::var(1), + Expr::app(Expr::reference(1, vec![]), Expr::reference(0, vec![])), + vec![], + vec![y, x], + vec![], + ); + let (a, _) = env1("Foo", &ca); + let (b, _) = env1("Foo", &cb); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert_eq!(labels(&c.fields), ["encoding"]); + assert!(c.rippled); + } + + /// Kind-change rows enter the map too: their dependents are rippled. + #[test] + fn ripple_kind_change_root_and_maps() { + let build = |axio: bool| { + let env = Env::new(); + let dc: Constant = if axio { + Constant::new(ConstantInfo::Axio(crate::constant::Axiom { + is_unsafe: false, + lvls: 0, + typ: Expr::var(3), + })) + } else { + defn_c(Expr::var(3), Expr::var(0)) + }; + let d_addr = store_canonical(&env, &dc); + let dep = defn_ref(9, d_addr.clone()); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("D"), Named::with_addr(d_addr)); + env.register_name(n("UsesD"), Named::with_addr(dep_addr)); + env + }; + let a = build(true); + let b = build(false); + let d = diff(&a, &b); + let kd = changed_by_name(&d, "D"); + assert!(!kd.rippled); + assert_eq!(labels(&kd.fields), ["kind"]); + let dep = changed_by_name(&d, "UsesD"); + assert!(dep.rippled, "kind rows must still enter the quotient map"); + } + + /// Two names sharing one old address, diverging in the new env: the + /// set-valued map explains both dependents. + #[test] + fn ripple_name_split_conflict() { + let a = Env::new(); + let x = defn_c(Expr::var(3), Expr::var(0)); + let x_addr = store_canonical(&a, &x); + a.register_name(n("P"), Named::with_addr(x_addr.clone())); + a.register_name(n("Q"), Named::with_addr(x_addr.clone())); + let dp = defn_ref(7, x_addr.clone()); + let dp_addr = store_canonical(&a, &dp); + a.register_name(n("DP"), Named::with_addr(dp_addr)); + let dq = defn_ref(8, x_addr); + let dq_addr = store_canonical(&a, &dq); + a.register_name(n("DQ"), Named::with_addr(dq_addr)); + + let b = Env::new(); + let y1 = defn_c(Expr::var(3), Expr::var(1)); + let y1_addr = store_canonical(&b, &y1); + let y2 = defn_c(Expr::var(3), Expr::var(2)); + let y2_addr = store_canonical(&b, &y2); + b.register_name(n("P"), Named::with_addr(y1_addr.clone())); + b.register_name(n("Q"), Named::with_addr(y2_addr.clone())); + let dp2 = defn_ref(7, y1_addr); + let dp2_addr = store_canonical(&b, &dp2); + b.register_name(n("DP"), Named::with_addr(dp2_addr)); + let dq2 = defn_ref(8, y2_addr); + let dq2_addr = store_canonical(&b, &dq2); + b.register_name(n("DQ"), Named::with_addr(dq2_addr)); + + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "P").rippled); + assert!(!changed_by_name(&d, "Q").rippled); + assert!(changed_by_name(&d, "DP").rippled); + assert!(changed_by_name(&d, "DQ").rippled); + } + + /// Ref univ arguments: same args over a mapped target → rippled; + /// changed arg arity (induced re-elaboration) → root. + #[test] + fn ripple_ref_univ_args() { + let build = |leaf_value: Arc, dep_univs: bool| { + let env = Env::new(); + let leaf = + Constant::new(ConstantInfo::Defn(mk_defn(1, Expr::var(3), leaf_value))); + let leaf_addr = store_canonical(&env, &leaf); + let dep = if dep_univs { + defn_ct( + Expr::var(9), + Expr::reference(0, vec![0, 1]), + vec![], + vec![leaf_addr.clone()], + vec![Univ::zero(), Univ::succ(Univ::zero())], + ) + } else { + defn_ct( + Expr::var(9), + Expr::reference(0, vec![0]), + vec![], + vec![leaf_addr.clone()], + vec![Univ::zero()], + ) + }; + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("Leaf"), Named::with_addr(leaf_addr)); + env.register_name(n("Dep"), Named::with_addr(dep_addr)); + env + }; + // Same univ args over a mapped target: rippled. + let a = build(Expr::var(0), false); + let b = build(Expr::var(1), false); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "Leaf").rippled); + assert!(changed_by_name(&d, "Dep").rippled); + + // Univ-argument arity changed at the use site: intrinsic (induced + // re-elaboration) → root, even though the target maps. + let a = build(Expr::var(0), false); + let b = build(Expr::var(1), true); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "Leaf").rippled); + let dep = changed_by_name(&d, "Dep"); + assert!(!dep.rippled); + assert_eq!(labels(&dep.fields), ["value"]); + } + + /// A dependency renamed (removed+added) never enters the map: its + /// dependents verdict root — fail-safe over-report. + #[test] + fn ripple_renamed_dep_is_root() { + let a = Env::new(); + let old = defn_c(Expr::var(3), Expr::var(0)); + let old_addr = store_canonical(&a, &old); + a.register_name(n("OldDep"), Named::with_addr(old_addr.clone())); + let dep = defn_ref(9, old_addr); + let dep_addr = store_canonical(&a, &dep); + a.register_name(n("User"), Named::with_addr(dep_addr)); + + let b = Env::new(); + let new = defn_c(Expr::var(3), Expr::var(1)); + let new_addr = store_canonical(&b, &new); + b.register_name(n("NewDep"), Named::with_addr(new_addr.clone())); + let dep2 = defn_ref(9, new_addr); + let dep2_addr = store_canonical(&b, &dep2); + b.register_name(n("User"), Named::with_addr(dep2_addr)); + + let d = diff(&a, &b); + assert_eq!(d.named_removed.len(), 1); + assert_eq!(d.named_added.len(), 1); + let user = changed_by_name(&d, "User"); + assert!(!user.rippled, "renamed dep is not in the map → root"); + } + + /// The third address site: `Expr::Prj` type targets quotient too. + #[test] + fn ripple_expr_prj_type_mapped() { + let build = |t_value: Arc| { + let env = Env::new(); + let t = defn_c(Expr::var(3), t_value); + let t_addr = store_canonical(&env, &t); + let dep = defn_ct( + Expr::var(9), + Expr::prj(0, 0, Expr::var(0)), + vec![], + vec![t_addr.clone()], + vec![], + ); + let dep_addr = store_canonical(&env, &dep); + env.register_name(n("T"), Named::with_addr(t_addr)); + env.register_name(n("UsesT"), Named::with_addr(dep_addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert!(!changed_by_name(&d, "T").rippled); + assert!(changed_by_name(&d, "UsesT").rippled); + } + + /// Changed projection coordinates target a different member: root. + #[test] + fn ripple_cprj_cidx_change_is_root() { + let block_a = Constant::new(ConstantInfo::Muts(vec![MutConst::Defn( + mk_defn(0, Expr::var(3), Expr::var(0)), + )])); + let block_b = Constant::new(ConstantInfo::Muts(vec![MutConst::Defn( + mk_defn(0, Expr::var(3), Expr::var(1)), + )])); + let a = Env::new(); + let ba = store_canonical(&a, &block_a); + let pa = store_canonical(&a, &ctor_proj_constant(0, 0, ba)); + a.register_name(n("C"), Named::with_addr(pa)); + let b = Env::new(); + let bb = store_canonical(&b, &block_b); + let pb = store_canonical(&b, &ctor_proj_constant(0, 1, bb)); + b.register_name(n("C"), Named::with_addr(pb)); + let d = diff(&a, &b); + let c = changed_by_name(&d, "C"); + assert_eq!(labels(&c.fields), ["cidx", "block"]); + assert!(!c.rippled); + } + + /// Alias rows spanning one (old, new) pair share one cached verdict. + #[test] + fn ripple_alias_rows_consistent() { + let build = |value: Arc| { + let env = Env::new(); + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + env.register_name(n("P"), Named::with_addr(addr.clone())); + env.register_name(n("Q"), Named::with_addr(addr)); + env + }; + let a = build(Expr::var(0)); + let b = build(Expr::var(1)); + let d = diff(&a, &b); + assert_eq!(d.named_changed.len(), 2); + let p = changed_by_name(&d, "P"); + let q = changed_by_name(&d, "Q"); + assert_eq!(p.rippled, q.rippled); + assert!(!p.rippled); + assert_eq!( + (p.old_addr.clone(), p.new_addr.clone()), + (q.old_addr.clone(), q.new_addr.clone()) + ); + } + + /// A lone changed leaf with no dependents: verdict root, and strict + /// fields identical to the pre-ripple behavior. + #[test] + fn ripple_single_root_no_map_hits() { + let (a, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(0))); + let (b, _) = env1("Foo", &defn_c(Expr::var(3), Expr::var(1))); + let d = diff(&a, &b); + let c = changed_by_name(&d, "Foo"); + assert!(!c.rippled); + assert_eq!(labels(&c.fields), ["value"]); + } } diff --git a/crates/ixon/src/lib.rs b/crates/ixon/src/lib.rs index 896e7a956..38243f2ad 100644 --- a/crates/ixon/src/lib.rs +++ b/crates/ixon/src/lib.rs @@ -33,7 +33,8 @@ pub use constant::{ }; #[cfg(not(target_arch = "riscv64"))] pub use diff::{ - EnvDiff, EnvStats, JoinProgress, NamedChange, diff_envs, diff_envs_with, + DiffPhase, EnvDiff, EnvStats, JoinProgress, NamedChange, diff_envs, + diff_envs_with, }; pub use env::{Env, Named}; pub use error::{CompileError, DecompileError, SerializeError}; From 1d3c00cf1618d2aca68f8f9ca2bcc744d0eaafa0 Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Fri, 10 Jul 2026 01:16:32 -0400 Subject: [PATCH 6/8] =?UTF-8?q?Ixon:=20memory-lean=20--meta=20(streaming?= =?UTF-8?q?=20=C2=A75=20sweep)=20+=20mmap=20path=20FFI=20for=20ix=20diff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --meta at mathlib scale previously OOM'd a 124 GiB machine (the full reader materializes every ConstantMeta on both sides). It now runs in 8 GiB peak / 67 s on the 744k-name mathlib pair. Streaming §5 sweep: both modes load via the lazy index; meta mode additionally merge-joins the two files' §5 named sections with a pair of streaming cursors (NamedMetaCursor), parsing each side's entry against its own §4 reverse index, comparing, and dropping — resident metadata is O(1) instead of everything-at-once. Raw §5 byte windows are NOT comparable across files (metadata name references are file-relative §4 indices; identical metadata serializes to different bytes over different name tables), so the sweep compares parsed, Address-valued ConstantMeta — pinned by a §4 index-shift soundness regression test and full-reader parity tests. §5 arrives in ascending name-hash order in both files (exactly Name's Ord), which makes the lockstep merge-join sound. LazyIndex now records the §5 offset and retains the §4 reverse index (~32 B/name); parse_lazy_index is otherwise unchanged. Engine API: diff_env_bytes (bytes-level, both modes lazy) and diff_envs_lazy over LazySide {env, index, data}; diff_envs/ diff_envs_with keep the in-env metadata path for materialized envs. DiffPhase gains MetaSweep (progress: "meta sweep: N/M (K differing)"). Mmap path FFI: rs_diff_env_files mmaps both files (Env::from_lazy_index_mmap keeps constant windows as zero-copy mmap slices via store_const_lazy_mmap) and rs_ixe_files_equal does the byte-equal fast path (metadata length check, then mmap memcmp). The CLI no longer reads files into Lean ByteArrays at all. The ByteArray-based rs_diff_envs stays for property tests, now also on the lazy path in both modes. Mathlib validation (v4.29.0 ↔ 8850ed93): --meta completes at 8 GiB peak / 67 s wall (sweep finds 7,998 differing names → 867 metadata-only rows + 7,131 changed rows with meta labels; structural counts identical to anon: 143,528 changed, 4,516 roots). Anon through the path FFI: 8 GiB peak (was 54 GiB heap-resident), report byte-identical to the ByteArray-path baseline. Identical-file fast path on a 3 GB input: 1.5 s. docs/Ixon.md gains a "Diffing environments" section (semantics, root/rippled, memory model). --- Cargo.lock | 1 + Ix/Cli/DiffCmd.lean | 37 +-- Ix/Ixon.lean | 18 ++ Tests/FFI/Ixon.lean | 25 +- crates/ffi/Cargo.toml | 1 + crates/ffi/src/lean_ixon/diff.rs | 306 ++++++++++++++++++------- crates/ixon/src/diff.rs | 376 +++++++++++++++++++++++++++++-- crates/ixon/src/env.rs | 66 ++++-- crates/ixon/src/lib.rs | 4 +- crates/ixon/src/serialize.rs | 69 +++++- docs/Ixon.md | 27 +++ 11 files changed, 794 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e44d6c16..b3ae9ab9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1802,6 +1802,7 @@ dependencies = [ "ixon", "ixvm-codegen", "lean-ffi", + "memmap2", "mimalloc", "multi-stark", "n0-error", diff --git a/Ix/Cli/DiffCmd.lean b/Ix/Cli/DiffCmd.lean index 799ea027f..6a65a551f 100644 --- a/Ix/Cli/DiffCmd.lean +++ b/Ix/Cli/DiffCmd.lean @@ -2,13 +2,15 @@ `ix diff `: structured diff of two serialized Ixon environments. - The diff itself is computed in Rust (`rs_diff_envs` → `ixon::diff`): - both files are parsed with the full reader and compared on anonymous - structure — names serve as join/display keys, constants compare by - content address with per-field classification (type/value/lvls/…, - `block.*` for projection targets, `"encoding"` when only the - representation moved). `--meta` additionally compares named metadata - (`ConstantMeta`/`original`). + The diff itself is computed in Rust (`rs_diff_env_files` → + `ixon::diff`): both files are memory-mapped and lazily parsed + (constant windows stay zero-copy mmap slices; `ConstantMeta` is never + bulk-materialized) and compared on anonymous structure — names serve + as join/display keys, constants compare by content address with + per-field classification (type/value/lvls/…, `block.*` for projection + targets, `"encoding"` when only the representation moved). `--meta` + additionally compares named metadata (`ConstantMeta`/`original`) via + a streaming merge-join over both files' §5 named sections. Every changed row carries a root-vs-rippled verdict: one edited constant re-addresses its whole reverse-dependency cone, so most @@ -128,17 +130,16 @@ def runDiffCmd (p : Cli.Parsed) : IO UInt32 := do IO.eprintln "error: --anon and --meta are mutually exclusive" return 2 let verbose := p.hasFlag "verbose" - let readBin (path : String) : IO (Option ByteArray) := - try pure (some (← IO.FS.readBinFile path)) - catch e => IO.eprintln s!"error: {path}: {e}"; pure none - let some oldBytes ← readBin oldPath | return 2 - let some newBytes ← readBin newPath | return 2 - if oldBytes == newBytes then - IO.println "identical" - return 0 - let d ← match Ixon.rsDiffEnvs oldBytes newBytes wantMeta with - | .error e => IO.eprintln s!"error: {e}"; return 2 - | .ok d => pure d + -- Byte-equal fast path and the diff itself both run over mmapped + -- files — nothing is read into Lean ByteArrays. + let d ← try + if ← Ixon.rsIxeFilesEqual oldPath newPath then + IO.println "identical" + return (0 : UInt32) + Ixon.rsDiffEnvFiles oldPath newPath wantMeta + catch e => + IO.eprintln s!"error: {e.toString}" + return (2 : UInt32) printStats oldPath d.statsA printStats newPath d.statsB if d.isEmpty then diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index 07c9eec3e..fe96b66d2 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -2363,6 +2363,24 @@ def rsDiffEnvs (a b : ByteArray) (compareMeta : Bool := false) : Except String EnvDiff := rsDiffEnvsFFI a b compareMeta +@[extern "rs_diff_env_files"] +opaque rsDiffEnvFilesFFI : @& String → @& String → Bool → IO EnvDiff + +/-- File-path variant of `rsDiffEnvs`: memory-maps both `.ixe` files + (constant windows stay zero-copy mmap slices backed by the OS page + cache) and diffs via the lazy reader in both modes — `ConstantMeta` + is never bulk-materialized; meta mode streams both §5 named + sections in a lockstep merge-join. The leanest diff path for + multi-GB envs. Failures surface as `IO` errors. -/ +def rsDiffEnvFiles (a b : String) (compareMeta : Bool := false) : + IO EnvDiff := + rsDiffEnvFilesFFI a b compareMeta + +/-- Byte-equality of two files: metadata length fast path, then an + mmap memcmp — no heap reads. -/ +@[extern "rs_ixe_files_equal"] +opaque rsIxeFilesEqual : @& String → @& String → IO Bool + /-! ### Env pack (`rs_pack_env`) -/ /-- Pack a value bundle in Rust: read the env at `envPath` (full diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index 0c855a483..2086e0cb1 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -378,7 +378,30 @@ def envDiffTests : TestSeq := let b := { envBase with assumptions := ({} : Std.HashSet Address).insert q |>.insert r } (runDiff a b).any fun d => - d.assumptionsAdded == #[r] && d.assumptionsRemoved == #[p]) + d.assumptionsAdded == #[r] && d.assumptionsRemoved == #[p]) ++ + -- The mmap file-path entry must agree with the bytes-based one, and + -- the byte-equality fast path must distinguish same vs different. + (TestSeq.individualIO "EnvDiff: file-based diff matches bytes-based" + none (do + let dir ← IO.FS.createTempDir + let pa := dir / "a.ixe" + let pb := dir / "b.ixe" + let bytesA := serEnv envBase + let bytesB := serEnv envValueChanged + IO.FS.writeBinFile pa bytesA + IO.FS.writeBinFile pb bytesB + let r ← try + let eqSame ← Ixon.rsIxeFilesEqual pa.toString pa.toString + let eqDiff ← Ixon.rsIxeFilesEqual pa.toString pb.toString + let dFiles ← Ixon.rsDiffEnvFiles pa.toString pb.toString false + let ok := eqSame && !eqDiff && + (match Ixon.rsDiffEnvs bytesA bytesB false with + | .ok dBytes => dFiles == dBytes + | .error _ => false) + pure (ok, if ok then none else some "file/bytes reports disagree") + catch e => pure (false, some e.toString) + IO.FS.removeDirAll dir + pure (r.1, 0, 0, r.2)) .done) /-- Self-diff over the pure-Lean writer's bytes must be empty in both modes (also exercises report marshaling on arbitrary inputs). -/ diff --git a/crates/ffi/Cargo.toml b/crates/ffi/Cargo.toml index 477add5af..1e02a9a6d 100644 --- a/crates/ffi/Cargo.toml +++ b/crates/ffi/Cargo.toml @@ -22,6 +22,7 @@ ix-compile = { workspace = true } ixon = { workspace = true } ix-kernel = { workspace = true } lean-ffi = { workspace = true } +memmap2 = { workspace = true } multi-stark = { workspace = true } mimalloc = { workspace = true } num-bigint = { workspace = true } diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs index c5ac2530e..36868ebec 100644 --- a/crates/ffi/src/lean_ixon/diff.rs +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -1,24 +1,32 @@ -//! `rs_diff_envs`: parse two serialized envs, diff them in Rust -//! (`ixon::diff`), and marshal the [`EnvDiff`] report to Lean -//! (`Ixon.EnvDiff`). +//! Env diff FFI: `rs_diff_envs` (two serialized byte arrays), +//! `rs_diff_env_files` (two file paths, memory-mapped), and +//! `rs_ixe_files_equal` (byte-equality fast path) — all marshaling the +//! [`EnvDiff`] report to Lean (`Ixon.EnvDiff`). //! -//! Anon mode (the default) loads via the memory-lean lazy-index path -//! (`parse_lazy_index` + `Env::from_lazy_index` — ConstantMeta is never -//! materialized); meta mode uses the full reader (`Env::get`, -//! preserving `Named.meta`/`original`). Large inputs get incremental -//! progress on stderr. +//! Both modes load via the memory-lean lazy-index path +//! (`parse_lazy_index` + `from_lazy_index`/`from_lazy_index_mmap` — +//! `ConstantMeta` is never bulk-materialized); meta mode additionally +//! streams both files' §5 named sections in a lockstep merge-join +//! (`ixon::diff::diff_envs_lazy`). The file-path entry keeps constant +//! windows as zero-copy mmap slices, so neither the 2× file bytes nor +//! the const-window copies of the ByteArray path are resident. Large +//! inputs get incremental progress on stderr. //! //! Names cross the boundary pre-rendered (`Name::pretty()`) and //! pre-sorted; addresses cross raw so Lean owns display formatting. +use std::sync::Arc; use std::time::Instant; use ix_common::address::Address; -use ixon::diff::{DiffPhase, EnvDiff, EnvStats, NamedChange, diff_envs_with}; -use ixon::env::Env as IxonEnv; +use ixon::diff::{ + DiffPhase, EnvDiff, EnvStats, JoinProgress, LazySide, NamedChange, + diff_envs_lazy, +}; +use ixon::env::{Env as IxonEnv, LazyIndex}; use lean_ffi::object::{ - LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanOption, - LeanOwned, LeanProd, LeanString, + LeanArray, LeanBool, LeanBorrowed, LeanByteArray, LeanExcept, LeanIOResult, + LeanOption, LeanOwned, LeanProd, LeanString, }; use crate::lean::{ @@ -151,86 +159,226 @@ impl LeanIxonEnvDiff { /// silent. const PROGRESS_MIN_BYTES: usize = 100 * 1024 * 1024; +/// Stderr line per progress event, tagged by phase. +fn print_progress(p: &JoinProgress) { + match p.phase { + DiffPhase::MetaSweep => eprintln!( + "[rs_diff_envs] meta sweep: {}/{} ({} differing so far)", + p.done, p.total, p.changed + ), + DiffPhase::NamedJoin => eprintln!( + "[rs_diff_envs] named join: {}/{} ({} changed so far)", + p.done, p.total, p.changed + ), + DiffPhase::RippleClassify => eprintln!( + "[rs_diff_envs] ripple pass: {}/{} ({} roots so far)", + p.done, p.total, p.changed + ), + } +} + +fn print_parse_start(which: &str, len: usize, progress: bool) { + if progress { + eprintln!( + "[rs_diff_envs] parsing {which} env ({} MB, lazy reader)...", + len / 1_000_000 + ); + } +} + +fn print_parse_done(which: &str, t: Instant, env: &IxonEnv, progress: bool) { + if progress { + eprintln!( + "[rs_diff_envs] {which} env parsed in {:.1}s ({} consts, {} named, {} blobs)", + t.elapsed().as_secs_f64(), + env.consts.len(), + env.named.len(), + env.blobs.len() + ); + } +} + +/// Lazy-parse one heap-backed side: index + env. +fn parse_side_bytes( + bytes: &[u8], + which: &str, + progress: bool, +) -> Result<(LazyIndex, IxonEnv), String> { + print_parse_start(which, bytes.len(), progress); + let t = Instant::now(); + let index = IxonEnv::parse_lazy_index(bytes) + .map_err(|e| format!("{which} input: {e}"))?; + let env = IxonEnv::from_lazy_index(&index, bytes) + .map_err(|e| format!("{which} input: {e}"))?; + print_parse_done(which, t, &env, progress); + Ok((index, env)) +} + +/// Run the shared lazy diff over two prepared sides and marshal. +fn run_diff( + a: LazySide<'_>, + b: LazySide<'_>, + want_meta: bool, + progress: bool, + who: &str, +) -> Result { + let t = Instant::now(); + let mut on_progress = |p: JoinProgress| { + if progress { + print_progress(&p); + } + }; + let d = diff_envs_lazy(a, b, want_meta, &mut on_progress)?; + if progress { + eprintln!("[{who}] diff computed in {:.1}s", t.elapsed().as_secs_f64()); + } + Ok(d) +} + /// FFI: diff two serialized envs. /// `(a b : ByteArray) → (meta : Bool) → Except String Ixon.EnvDiff`. -/// Pure result; reader choice (lazy-index vs full) follows `meta` — -/// see the module docs. For large inputs, progress goes to stderr. +/// Pure result; both modes use the lazy reader (meta mode adds the +/// streaming §5 sweep). For large inputs, progress goes to stderr. #[unsafe(no_mangle)] pub extern "C" fn rs_diff_envs( a: LeanByteArray>, b: LeanByteArray>, meta: LeanBool>, ) -> LeanExcept { - let progress = a.as_bytes().len() >= PROGRESS_MIN_BYTES - || b.as_bytes().len() >= PROGRESS_MIN_BYTES; + let (a_bytes, b_bytes) = (a.as_bytes(), b.as_bytes()); + let progress = + a_bytes.len() >= PROGRESS_MIN_BYTES || b_bytes.len() >= PROGRESS_MIN_BYTES; let want_meta = meta.to_bool(); - // Anon mode loads via the lazy index: constant byte-windows + - // name→addr + §3 hints + §6 comms, never materializing ConstantMeta - // — at mathlib scale the full reader's metadata costs tens of GB and - // OOMs a 128 GB machine when two envs are held at once. Meta mode - // needs `Named.meta`/`original`, so it pays for the full reader. - let parse = |bytes: &[u8], which: &str| -> Result { - let reader = if want_meta { "full" } else { "anon" }; - if progress { - eprintln!( - "[rs_diff_envs] parsing {which} env ({} MB, {reader} reader)...", - bytes.len() / 1_000_000 - ); - } - let t = Instant::now(); - let env = if want_meta { - let mut cursor: &[u8] = bytes; - IxonEnv::get(&mut cursor).map_err(|e| format!("{which} input: {e}"))? - } else { - let index = IxonEnv::parse_lazy_index(bytes) - .map_err(|e| format!("{which} input: {e}"))?; - IxonEnv::from_lazy_index(&index, bytes) - .map_err(|e| format!("{which} input: {e}"))? - }; - if progress { - eprintln!( - "[rs_diff_envs] {which} env parsed in {:.1}s ({} consts, {} named, {} blobs)", - t.elapsed().as_secs_f64(), - env.consts.len(), - env.named.len(), - env.blobs.len() - ); - } - Ok(env) - }; - let env_a = match parse(a.as_bytes(), "first") { - Ok(env) => env, - Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), - }; - let env_b = match parse(b.as_bytes(), "second") { - Ok(env) => env, + let sides = parse_side_bytes(a_bytes, "first", progress) + .and_then(|a| Ok((a, parse_side_bytes(b_bytes, "second", progress)?))); + let ((ia, ea), (ib, eb)) = match sides { + Ok(s) => s, Err(e) => return LeanExcept::error_string(&format!("rs_diff_envs: {e}")), }; + match run_diff( + LazySide { env: &ea, index: &ia, data: a_bytes }, + LazySide { env: &eb, index: &ib, data: b_bytes }, + want_meta, + progress, + "rs_diff_envs", + ) { + Ok(d) => LeanExcept::ok(LeanIxonEnvDiff::build(&d)), + Err(e) => LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + } +} + +/// Memory-map a file, guarding against a concurrent length change. +fn mmap_file(path: &str, which: &str) -> Result, String> { + let file = std::fs::File::open(path) + .map_err(|e| format!("{which} input {path}: {e}"))?; + let expected = + file.metadata().map_err(|e| format!("{which} input {path}: {e}"))?.len(); + let mmap = unsafe { memmap2::Mmap::map(&file) } + .map_err(|e| format!("{which} input {path}: mmap failed: {e}"))?; + if mmap.len() as u64 != expected { + return Err(format!( + "{which} input {path}: mapped length {} != metadata length {expected}", + mmap.len() + )); + } + Ok(Arc::new(mmap)) +} + +/// Lazy-parse one mmap-backed side: constant windows stay zero-copy. +fn parse_side_mmap( + mmap: &Arc, + which: &str, + progress: bool, +) -> Result<(LazyIndex, IxonEnv), String> { + print_parse_start(which, mmap.len(), progress); let t = Instant::now(); - let mut on_progress = |p: ixon::diff::JoinProgress| { - if progress { - match p.phase { - DiffPhase::NamedJoin => eprintln!( - "[rs_diff_envs] named join: {}/{} ({} changed so far)", - p.done, p.total, p.changed - ), - DiffPhase::RippleClassify => eprintln!( - "[rs_diff_envs] ripple pass: {}/{} ({} roots so far)", - p.done, p.total, p.changed - ), - } - } + let index = IxonEnv::parse_lazy_index(&mmap[..]) + .map_err(|e| format!("{which} input: {e}"))?; + let env = IxonEnv::from_lazy_index_mmap(&index, mmap) + .map_err(|e| format!("{which} input: {e}"))?; + print_parse_done(which, t, &env, progress); + Ok((index, env)) +} + +/// FFI: diff two `.ixe` files by path, memory-mapped. +/// `(a b : String) → (meta : Bool) → IO Ixon.EnvDiff`. +/// Constant windows are zero-copy mmap slices, so neither the file +/// bytes nor const-window copies are heap-resident — the leanest diff +/// path for multi-GB envs. Errors surface as `IO` errors. +#[unsafe(no_mangle)] +pub extern "C" fn rs_diff_env_files( + a_path: LeanString>, + b_path: LeanString>, + meta: LeanBool>, +) -> LeanIOResult { + let want_meta = meta.to_bool(); + let a_path = a_path.to_string(); + let b_path = b_path.to_string(); + let sides = mmap_file(&a_path, "first") + .and_then(|ma| Ok((ma, mmap_file(&b_path, "second")?))); + let (ma, mb) = match sides { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")); + }, }; - match diff_envs_with(&env_a, &env_b, want_meta, &mut on_progress) { - Ok(d) => { - if progress { - eprintln!( - "[rs_diff_envs] diff computed in {:.1}s", - t.elapsed().as_secs_f64() - ); - } - LeanExcept::ok(LeanIxonEnvDiff::build(&d)) + let progress = + ma.len() >= PROGRESS_MIN_BYTES || mb.len() >= PROGRESS_MIN_BYTES; + let parsed = parse_side_mmap(&ma, "first", progress) + .and_then(|a| Ok((a, parse_side_mmap(&mb, "second", progress)?))); + let ((ia, ea), (ib, eb)) = match parsed { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")); }, - Err(e) => LeanExcept::error_string(&format!("rs_diff_envs: {e}")), + }; + match run_diff( + LazySide { env: &ea, index: &ia, data: &ma[..] }, + LazySide { env: &eb, index: &ib, data: &mb[..] }, + want_meta, + progress, + "rs_diff_env_files", + ) { + Ok(d) => LeanIOResult::ok(LeanIxonEnvDiff::build(&d)), + Err(e) => LeanIOResult::error_string(&format!("rs_diff_env_files: {e}")), } } + +/// FFI: byte-equality of two files. `(a b : String) → IO Bool`. +/// Length fast path via metadata, then an mmap memcmp — no heap reads. +#[unsafe(no_mangle)] +pub extern "C" fn rs_ixe_files_equal( + a_path: LeanString>, + b_path: LeanString>, +) -> LeanIOResult { + let a_path = a_path.to_string(); + let b_path = b_path.to_string(); + let len = |path: &str, which: &str| -> Result { + std::fs::metadata(path) + .map(|m| m.len()) + .map_err(|e| format!("{which} input {path}: {e}")) + }; + let lens = + len(&a_path, "first").and_then(|la| Ok((la, len(&b_path, "second")?))); + let (la, lb) = match lens { + Ok(s) => s, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_ixe_files_equal: {e}")); + }, + }; + let eq = if la != lb { + false + } else if la == 0 { + true // two empty files; mmap of a zero-length file errors + } else { + let maps = mmap_file(&a_path, "first") + .and_then(|ma| Ok((ma, mmap_file(&b_path, "second")?))); + match maps { + Ok((ma, mb)) => ma[..] == mb[..], + Err(e) => { + return LeanIOResult::error_string(&format!("rs_ixe_files_equal: {e}")); + }, + } + }; + LeanIOResult::ok(LeanOwned::box_usize(usize::from(eq))) +} diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs index 60c2a22ef..70d2911c7 100644 --- a/crates/ixon/src/diff.rs +++ b/crates/ixon/src/diff.rs @@ -54,6 +54,7 @@ //! prefixed) churn as one removed+added pair per changed block; display //! layers may group them. +use std::cmp::Ordering; use std::sync::Arc; use rustc_hash::{FxHashMap, FxHashSet}; @@ -65,8 +66,9 @@ use super::constant::{ Constant, ConstantInfo, Constructor, Definition, Inductive, MutConst, Recursor, }; -use super::env::{Env, Named}; +use super::env::{Env, LazyIndex, Named}; use super::expr::Expr; +use super::serialize::NamedMetaCursor; use super::univ::Univ; /// Per-env entity counts, reported for both inputs so display layers can @@ -829,13 +831,11 @@ fn classify_named_change( name: &Name, na: &Named, nb: &Named, - meta: bool, + meta_fields: Vec, ) -> Result { let ca = materialize_const(env_a, &na.addr, name, "first")?; let cb = materialize_const(env_b, &nb.addr, name, "second")?; let fields = classify_constants(env_a, &ca, env_b, &cb, None)?; - let meta_fields = - if meta { meta_component_labels(na, nb) } else { Vec::new() }; Ok(NamedChange { name: name.pretty(), old_addr: na.addr.clone(), @@ -871,6 +871,9 @@ fn sort_by_name(v: &mut [(String, Name, T)]) { /// Which diff pass a [`JoinProgress`] event reports. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DiffPhase { + /// Meta mode only, before the join: the streaming §5 metadata sweep + /// (lockstep merge-join of both files' named sections). + MetaSweep, /// Pass 1: the named join (change detection + strict classification). NamedJoin, /// Pass 2: re-classification of changed rows under the ripple @@ -878,25 +881,25 @@ pub enum DiffPhase { RippleClassify, } -/// Coarse progress for the two passes whose cost scales with the number -/// of *changed* names (each one materializes two constants and walks -/// their expressions). Parses happen before [`diff_envs`] and are the -/// caller's to report. +/// Coarse progress for the passes whose cost scales with entry counts. +/// Parses happen before [`diff_envs`] and are the caller's to report. #[derive(Clone, Copy, Debug)] pub struct JoinProgress { /// The pass this event reports. `done` counts reset between phases. pub phase: DiffPhase, + /// [`DiffPhase::MetaSweep`]: §5 entries merged (max of both sides). /// [`DiffPhase::NamedJoin`]: first-env named entries processed. /// [`DiffPhase::RippleClassify`]: changed rows verdicted. pub done: usize, /// Total entries of the reporting phase. pub total: usize, + /// [`DiffPhase::MetaSweep`]: names with differing metadata so far. /// [`DiffPhase::NamedJoin`]: changed names classified so far. /// [`DiffPhase::RippleClassify`]: roots found so far. pub changed: usize, } -/// Named-join entries between progress callbacks in [`diff_envs_with`]. +/// Named-join / meta-sweep entries between progress callbacks. const JOIN_PROGRESS_STRIDE: usize = 100_000; /// Changed rows between pass-2 progress callbacks — one event every @@ -904,21 +907,171 @@ const JOIN_PROGRESS_STRIDE: usize = 100_000; /// constant pair, so rows are ~10× slower than join probes). const RIPPLE_PROGRESS_STRIDE: usize = 10_000; +/// Where metadata comparisons come from. +/// +/// The full-reader path holds complete `Named` values in the envs +/// (`InEnv`); the memory-lean bytes/mmap path never materializes them +/// in bulk, so the streaming §5 sweep precomputes labels per name +/// (`Sweep`); anon mode compares no metadata at all (`None`). +enum MetaSource<'m> { + None, + InEnv, + Sweep(&'m FxHashMap>), +} + +impl MetaSource<'_> { + fn labels(&self, name: &Name, na: &Named, nb: &Named) -> Vec { + match self { + MetaSource::None => Vec::new(), + MetaSource::InEnv => meta_component_labels(na, nb), + MetaSource::Sweep(m) => m.get(name).cloned().unwrap_or_default(), + } + } +} + /// Diff two environments. `meta = false` (anon mode, the default) /// compares only anonymous structure; `meta = true` additionally -/// compares `Named` metadata content. See the module docs. +/// compares `Named` metadata content held in the envs (full-reader +/// path — for the memory-lean bytes path see [`diff_env_bytes`]). See +/// the module docs. pub fn diff_envs(a: &Env, b: &Env, meta: bool) -> Result { diff_envs_with(a, b, meta, &mut |_| {}) } /// [`diff_envs`] with a progress callback: `on_progress` fires every -/// `JOIN_PROGRESS_STRIDE` named-join entries and once when the join -/// completes (so the final event always carries `done == total`). +/// `JOIN_PROGRESS_STRIDE` named-join entries and once when each phase +/// completes (final events always carry `done == total`). pub fn diff_envs_with( a: &Env, b: &Env, meta: bool, on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + let source = if meta { MetaSource::InEnv } else { MetaSource::None }; + diff_envs_impl(a, b, &source, on_progress) +} + +/// One side of a lazy-path diff: the env built from `index` over `data` +/// (via [`Env::from_lazy_index`] or [`Env::from_lazy_index_mmap`]). +#[derive(Clone, Copy)] +pub struct LazySide<'a> { + pub env: &'a Env, + pub index: &'a LazyIndex, + pub data: &'a [u8], +} + +/// Memory-lean bytes-level diff (the `ix diff` path): lazy-index +/// structural load plus, in meta mode, the streaming §5 metadata sweep +/// — `ConstantMeta` is never bulk-materialized on either side. +pub fn diff_env_bytes( + a: &[u8], + b: &[u8], + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + let ia = Env::parse_lazy_index(a)?; + let ib = Env::parse_lazy_index(b)?; + let ea = Env::from_lazy_index(&ia, a)?; + let eb = Env::from_lazy_index(&ib, b)?; + diff_envs_lazy( + LazySide { env: &ea, index: &ia, data: a }, + LazySide { env: &eb, index: &ib, data: b }, + meta, + on_progress, + ) +} + +/// Core of the bytes/mmap paths: both sides already lazily loaded. In +/// meta mode, runs the streaming §5 sweep first (metadata labels per +/// name, parse-compare-drop), then the structural join consuming them. +pub fn diff_envs_lazy( + a: LazySide<'_>, + b: LazySide<'_>, + meta: bool, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result { + if meta { + let map = sweep_meta(&a, &b, on_progress)?; + diff_envs_impl(a.env, b.env, &MetaSource::Sweep(&map), on_progress) + } else { + diff_envs_impl(a.env, b.env, &MetaSource::None, on_progress) + } +} + +/// Streaming §5 metadata sweep: lockstep merge-join of both files' +/// named sections (canonical ascending name-hash order — exactly +/// `Name`'s `Ord`), parsing each side's entry against its own §4 +/// reverse index, comparing, and dropping. Returns component labels +/// for every joined name whose metadata differs — the addr-equal ones +/// become `named_meta_only` rows, the addr-changed ones `meta_fields`. +fn sweep_meta( + a: &LazySide<'_>, + b: &LazySide<'_>, + on_progress: &mut dyn FnMut(JoinProgress), +) -> Result>, String> { + let mut cur_a = NamedMetaCursor::open(a.data, a.index)?; + let mut cur_b = NamedMetaCursor::open(b.data, b.index)?; + let an = &a.index.named; + let bn = &b.index.named; + let total = an.len().max(bn.len()); + let mut out: FxHashMap> = FxHashMap::default(); + let (mut i, mut j) = (0usize, 0usize); + let mut fired = 0usize; + while i < an.len() && j < bn.len() { + match an[i].name.cmp(&bn[j].name) { + Ordering::Less => { + cur_a.next_entry()?; + i += 1; + }, + Ordering::Greater => { + cur_b.next_entry()?; + j += 1; + }, + Ordering::Equal => { + let (_, na) = cur_a + .next_entry()? + .ok_or("sweep_meta: first cursor exhausted early")?; + let (_, nb) = cur_b + .next_entry()? + .ok_or("sweep_meta: second cursor exhausted early")?; + // Cursor and index walked the same section — desync means a bug. + if na.addr != an[i].addr || nb.addr != bn[j].addr { + return Err("sweep_meta: cursor desynced from index".to_string()); + } + let labels = meta_component_labels(&na, &nb); + if !labels.is_empty() { + out.insert(an[i].name.clone(), labels); + } + i += 1; + j += 1; + }, + } + let done = i.max(j); + if done / JOIN_PROGRESS_STRIDE > fired && done < total { + fired = done / JOIN_PROGRESS_STRIDE; + on_progress(JoinProgress { + phase: DiffPhase::MetaSweep, + done, + total, + changed: out.len(), + }); + } + } + // Unjoined tails carry no comparison work — cursors just drop. + on_progress(JoinProgress { + phase: DiffPhase::MetaSweep, + done: total, + total, + changed: out.len(), + }); + Ok(out) +} + +fn diff_envs_impl( + a: &Env, + b: &Env, + source: &MetaSource<'_>, + on_progress: &mut dyn FnMut(JoinProgress), ) -> Result { let mut d = EnvDiff { stats_a: env_stats(a), @@ -944,14 +1097,13 @@ pub fn diff_envs_with( Some(nb_ref) => { let nb = nb_ref.value(); if na.addr == nb.addr { - if meta { - let labels = meta_component_labels(na, nb); - if !labels.is_empty() { - meta_only.push((name.pretty(), name.clone(), labels)); - } + let labels = source.labels(name, na, nb); + if !labels.is_empty() { + meta_only.push((name.pretty(), name.clone(), labels)); } } else { - let change = classify_named_change(a, b, name, na, nb, meta)?; + let meta_fields = source.labels(name, na, nb); + let change = classify_named_change(a, b, name, na, nb, meta_fields)?; let slot = addr_map.entry(na.addr.clone()).or_default(); if !slot.contains(&nb.addr) { slot.push(nb.addr.clone()); @@ -2159,4 +2311,192 @@ mod tests { assert!(!c.rippled); assert_eq!(labels(&c.fields), ["value"]); } + + // ========================================================================== + // Streaming §5 meta sweep (diff_env_bytes) + // ========================================================================== + + /// Register `name`'s component in `env.names`; returns its address. + fn register_component(env: &Env, name: &Name) -> Address { + let addr = Address::from_blake3_hash(*name.get_hash()); + env.names.insert(addr.clone(), name.clone()); + addr + } + + fn def_meta(name_addr: Address, hint: u32) -> ConstantMeta { + ConstantMeta::new(ConstantMetaInfo::Def { + name: name_addr, + lvls: vec![], + hints: ReducibilityHints::Regular(hint), + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }) + } + + fn full_read(bytes: &[u8]) -> Env { + let mut cur = bytes; + Env::get(&mut cur).expect("full read failed") + } + + /// The streaming §5 sweep must reproduce the full reader's meta-mode + /// report exactly: `named_meta_only`, `meta_fields` on changed rows, + /// and everything structural. + #[test] + fn meta_sweep_matches_full_reader() { + let build = |foo_value: Arc, + foo_hint: u32, + monly_hint: u32, + foo_original: bool| + -> Vec { + let env = Env::new(); + let foo = n("Foo"); + let foo_comp = register_component(&env, &foo); + let c = defn_c(Expr::var(3), foo_value); + let addr = store_canonical(&env, &c); + let mut foo_named = + Named::new(addr.clone(), def_meta(foo_comp, foo_hint)); + if foo_original { + foo_named.set_original(addr, ConstantMeta::default()); + } + env.register_name(foo.clone(), foo_named); + // Same const both sides; only metadata differs → named_meta_only. + let monly = n("MetaOnly"); + let monly_comp = register_component(&env, &monly); + let stable = defn_c(Expr::var(7), Expr::var(0)); + let stable_addr = store_canonical(&env, &stable); + env.register_name( + monly.clone(), + Named::new(stable_addr, def_meta(monly_comp, monly_hint)), + ); + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let ba = build(Expr::var(0), 1, 3, false); + let bb = build(Expr::var(1), 2, 4, true); + + let via_full = + diff_envs(&full_read(&ba), &full_read(&bb), true).expect("full diff"); + let mut events: Vec = Vec::new(); + let via_sweep = diff_env_bytes(&ba, &bb, true, &mut |p| events.push(p)) + .expect("sweep diff"); + assert_eq!(via_full, via_sweep); + // The pair genuinely exercises both meta categories. + assert_eq!(via_full.named_meta_only.len(), 1); + assert_eq!(via_full.named_meta_only[0].0, "MetaOnly"); + let foo_row = + via_full.named_changed.iter().find(|c| c.name == "Foo").unwrap(); + assert!(!foo_row.meta_fields.is_empty()); + // The sweep phase fired and completed before the join started. + assert_eq!(events.first().map(|p| p.phase), Some(DiffPhase::MetaSweep)); + let last_sweep = + events.iter().rfind(|p| p.phase == DiffPhase::MetaSweep).unwrap(); + assert_eq!(last_sweep.done, last_sweep.total); + + // Anon parity through the bytes path too. + let anon_full = + diff_envs(&full_read(&ba), &full_read(&bb), false).expect("anon full"); + let anon_sweep = + diff_env_bytes(&ba, &bb, false, &mut |_| {}).expect("anon sweep"); + assert_eq!(anon_full, anon_sweep); + // Self-diff via the bytes path is empty in both modes. + assert!(diff_env_bytes(&ba, &ba, true, &mut |_| {}).unwrap().is_empty()); + assert!(diff_env_bytes(&ba, &ba, false, &mut |_| {}).unwrap().is_empty()); + } + + /// Metadata name references are file-relative §4 indices, so raw §5 + /// windows differ across files whose name tables differ — the sweep + /// must still see logically identical metadata as equal (it compares + /// parsed, Address-valued values, never bytes). + #[test] + fn meta_sweep_ignores_name_index_shift() { + let foo = n("Foo"); + let foo_comp = Address::from_blake3_hash(*foo.get_hash()); + // A name whose component address sorts BEFORE Foo's: its presence + // in env B shifts Foo's §4 index, re-encoding Foo's (identical) + // metadata over different indices. + let extra = (0..) + .map(|i| n(&format!("X{i}"))) + .find(|nm| Address::from_blake3_hash(*nm.get_hash()) < foo_comp) + .expect("some candidate component sorts before Foo"); + let build = |with_extra: bool| -> Vec { + let env = Env::new(); + register_component(&env, &foo); + let c = defn_c(Expr::var(3), Expr::var(0)); + let addr = store_canonical(&env, &c); + env.register_name( + foo.clone(), + Named::new(addr, def_meta(foo_comp.clone(), 1)), + ); + if with_extra { + register_component(&env, &extra); + let c2 = defn_c(Expr::var(8), Expr::var(2)); + let a2 = store_canonical(&env, &c2); + env.register_name(extra.clone(), Named::with_addr(a2)); + } + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let ba = build(false); + let bb = build(true); + let d = diff_env_bytes(&ba, &bb, true, &mut |_| {}).expect("sweep diff"); + assert_eq!(d.named_added.len(), 1); + assert!( + d.named_meta_only.is_empty(), + "index shift misread as a metadata diff: {d:?}" + ); + assert!(d.named_changed.is_empty()); + // And the full reader agrees. + assert_eq!( + d, + diff_envs(&full_read(&ba), &full_read(&bb), true).expect("full diff") + ); + } + + /// Mmap-backed lazy sides produce the same report as heap-backed + /// ones (the `rs_diff_env_files` path). + #[test] + fn mmap_lazy_side_matches_heap() { + let build = |value: Arc| -> Vec { + let env = Env::new(); + let foo = n("Foo"); + register_component(&env, &foo); + let c = defn_c(Expr::var(3), value); + let addr = store_canonical(&env, &c); + env.register_name(foo, Named::with_addr(addr)); + let mut bytes = Vec::new(); + env.put(&mut bytes).expect("put failed"); + bytes + }; + let (ba, bb) = (build(Expr::var(0)), build(Expr::var(1))); + + let path = std::env::temp_dir() + .join(format!("ixon-diff-mmap-test-{}.ixe", std::process::id())); + std::fs::write(&path, &ba).expect("write temp"); + let file = std::fs::File::open(&path).expect("open temp"); + let mmap = + Arc::new(unsafe { memmap2::Mmap::map(&file) }.expect("mmap temp")); + let ia = Env::parse_lazy_index(&mmap[..]).expect("lazy index (mmap)"); + let ea = Env::from_lazy_index_mmap(&ia, &mmap).expect("from mmap"); + + let ib = Env::parse_lazy_index(&bb).expect("lazy index (heap)"); + let eb = Env::from_lazy_index(&ib, &bb).expect("from heap"); + + let via_mmap = diff_envs_lazy( + LazySide { env: &ea, index: &ia, data: &mmap[..] }, + LazySide { env: &eb, index: &ib, data: &bb }, + true, + &mut |_| {}, + ) + .expect("mmap-side diff"); + let via_heap = + diff_env_bytes(&ba, &bb, true, &mut |_| {}).expect("heap diff"); + assert_eq!(via_mmap, via_heap); + assert_eq!(via_heap.named_changed.len(), 1); + std::fs::remove_file(&path).ok(); + } } diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index dae1b7f1a..0043b9333 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -208,6 +208,13 @@ pub struct LazyIndex { pub hints: Vec<(Address, ReducibilityHints)>, /// §6 comms (tiny in practice; empty for compile-produced envs). pub comms: Vec<(Address, Comm)>, + /// Byte offset (within the parsed buffer) of §5's entry count — where + /// a [`crate::serialize::NamedMetaCursor`] starts its streaming walk. + pub named_section_offset: usize, + /// §4 positional index → name-component address, retained so §5 + /// entries can be re-parsed standalone (`get_named_indexed`) without + /// re-walking §4. ~32 B per name. + pub name_reverse_index: crate::metadata::NameReverseIndex, } /// The Ixon environment. @@ -373,22 +380,53 @@ impl Env { index: &LazyIndex, data: &[u8], ) -> Result { - let mut env = Env::new(); + let env = Env::new(); for c in &index.consts { - let end = c - .offset - .checked_add(c.len) - .filter(|e| *e <= data.len()) - .ok_or_else(|| { - format!( - "from_lazy_index: constant window [{}, +{}) out of bounds ({} bytes)", - c.offset, - c.len, - data.len() - ) - })?; + let end = Self::lazy_slice_end(c, data.len(), "from_lazy_index")?; env.store_const_lazy(c.addr.clone(), Arc::from(&data[c.offset..end])); } + Ok(Self::fill_from_lazy_index(env, index)) + } + + /// [`Env::from_lazy_index`] over a memory-mapped buffer: constant + /// windows stay zero-copy mmap slices (the OS page cache backs them) + /// instead of heap copies. `mmap` must be the exact buffer + /// [`Env::parse_lazy_index`] walked. + #[cfg(not(target_arch = "riscv64"))] + pub fn from_lazy_index_mmap( + index: &LazyIndex, + mmap: &Arc, + ) -> Result { + let env = Env::new(); + for c in &index.consts { + Self::lazy_slice_end(c, mmap.len(), "from_lazy_index_mmap")?; + env.store_const_lazy_mmap( + c.addr.clone(), + Arc::clone(mmap), + c.offset, + c.len, + ); + } + Ok(Self::fill_from_lazy_index(env, index)) + } + + #[cfg(not(target_arch = "riscv64"))] + fn lazy_slice_end( + c: &LazyConstSlice, + data_len: usize, + who: &str, + ) -> Result { + c.offset.checked_add(c.len).filter(|e| *e <= data_len).ok_or_else(|| { + format!( + "{who}: constant window [{}, +{}) out of bounds ({data_len} bytes)", + c.offset, c.len, + ) + }) + } + + /// The non-const parts shared by both `from_lazy_index` variants. + #[cfg(not(target_arch = "riscv64"))] + fn fill_from_lazy_index(mut env: Env, index: &LazyIndex) -> Env { for n in &index.named { env.register_name(n.name.clone(), Named::with_addr(n.addr.clone())); } @@ -403,7 +441,7 @@ impl Env { } env.main = index.main.clone(); env.assumptions = index.assumptions.iter().cloned().collect(); - Ok(env) + env } /// Get a constant by address, materializing on demand. diff --git a/crates/ixon/src/lib.rs b/crates/ixon/src/lib.rs index 38243f2ad..bb9a900e1 100644 --- a/crates/ixon/src/lib.rs +++ b/crates/ixon/src/lib.rs @@ -33,8 +33,8 @@ pub use constant::{ }; #[cfg(not(target_arch = "riscv64"))] pub use diff::{ - DiffPhase, EnvDiff, EnvStats, JoinProgress, NamedChange, diff_envs, - diff_envs_with, + DiffPhase, EnvDiff, EnvStats, JoinProgress, LazySide, NamedChange, + diff_env_bytes, diff_envs, diff_envs_lazy, diff_envs_with, }; pub use env::{Env, Named}; pub use error::{CompileError, DecompileError, SerializeError}; diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index efe2121f9..cb41fc2d2 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -1215,6 +1215,62 @@ pub fn get_named_indexed( Ok(named) } +/// Streaming cursor over an env's §5 named entries: parse one `Named` +/// at a time against this file's own §4 reverse index, hand it out, +/// drop it. Entries arrive in the file's canonical ascending +/// name-hash order (which is exactly `Name`'s `Ord`), so two cursors +/// merge-join by name. +/// +/// This exists for the diff meta sweep: raw §5 windows are NOT +/// comparable across files (metadata name references are file-relative +/// §4 indices — identical metadata serializes to different bytes over +/// different name tables), but parsed [`Named`] values are +/// Address-valued and file-independent. Streaming keeps resident +/// metadata O(1) instead of the full reader's everything-at-once. +pub struct NamedMetaCursor<'a> { + buf: &'a [u8], + remaining: u64, + rev: &'a NameReverseIndex, +} + +impl<'a> NamedMetaCursor<'a> { + /// Open a cursor at `index.named_section_offset` within `data` — the + /// exact buffer [`Env::parse_lazy_index`] produced `index` from. + pub fn open( + data: &'a [u8], + index: &'a LazyIndex, + ) -> Result, String> { + let mut buf = data.get(index.named_section_offset..).ok_or_else(|| { + format!( + "NamedMetaCursor: §5 offset {} out of bounds ({} bytes)", + index.named_section_offset, + data.len() + ) + })?; + let remaining = get_u64(&mut buf)?; + if remaining as usize != index.named.len() { + return Err(format!( + "NamedMetaCursor: §5 count {} disagrees with the index ({} entries)", + remaining, + index.named.len() + )); + } + Ok(NamedMetaCursor { buf, remaining, rev: &index.name_reverse_index }) + } + + /// Parse the next entry: `(name-component hash, Named)`; `None` once + /// the section is exhausted. + pub fn next_entry(&mut self) -> Result, String> { + if self.remaining == 0 { + return Ok(None); + } + self.remaining -= 1; + let name_addr = get_address(&mut self.buf)?; + let named = get_named_indexed(&mut self.buf, self.rev)?; + Ok(Some((name_addr, named))) + } +} + // ============================================================================ // Env serialization // ============================================================================ @@ -1894,8 +1950,10 @@ impl Env { let hints_map: FxHashMap = index.hints.iter().cloned().collect(); - // Section 4: Names — parsed to build the index for metadata decoding, then - // dropped (the check never consults the name component table). + // Section 4: Names — parsed to build the index for metadata + // decoding. The reverse index is retained on the LazyIndex so §5 + // entries can be re-parsed standalone later (the diff meta sweep); + // the Address→Name lookup is dropped (LazyNamed carries the Names). let num_names = get_u64(&mut buf)?; let mut names_lookup: FxHashMap = FxHashMap::default(); let mut name_reverse_index: NameReverseIndex = @@ -1908,14 +1966,17 @@ impl Env { name_reverse_index.push(addr.clone()); names_lookup.insert(addr, name); } + index.name_reverse_index = name_reverse_index; // Section 5: Named — keep `name → addr` plus the §3 hint for that // address; the full ConstantMeta (arena, CallSite, ...) is parsed - // then discarded. + // then discarded. The section offset is recorded so a + // [`NamedMetaCursor`] can re-walk it without re-parsing §1–§4. + index.named_section_offset = data.len() - buf.len(); let num_named = get_u64(&mut buf)?; for _ in 0..num_named { let name_addr = get_address(&mut buf)?; - let named = get_named_indexed(&mut buf, &name_reverse_index)?; + let named = get_named_indexed(&mut buf, &index.name_reverse_index)?; let name = names_lookup.get(&name_addr).cloned().ok_or_else(|| { format!("parse_lazy_index: missing name for addr {:?}", name_addr) })?; diff --git a/docs/Ixon.md b/docs/Ixon.md index 696bddfc8..e6dbe66ad 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -946,6 +946,33 @@ reads with the full reader so display metadata survives the prune; compile once, pack many. (`ix shard extract` is the non-bundle sibling: a general sub-env for the kernel-check pipeline, no `main` root.) +### Diffing environments + +``` +lake exe ix diff [--anon | --meta] [--verbose] +``` + +`ix diff` (engine: `ixon::diff`) joins two envs on names: a name +"changed" ⇔ its constant address changed, with per-field +classification (`type`/`value`/`lvls`/…, `block.*` through projection +descent, `"encoding"` for pure representation churn). Because one +edited constant re-addresses its whole reverse-dependency cone, every +changed row also carries a **root vs rippled** verdict: changed pairs +are re-classified under the quotient of all changed rows' old→new +address mapping, and rows fully explained by dependency re-addressing +are `rippled` (hidden by default; `--verbose` lists). A 5-day mathlib +window classifies 143k changed names into ~4.5k roots. + +Memory: both files are mmap'd and lazily parsed in both modes +(constant windows stay zero-copy; `ConstantMeta` is never +bulk-materialized). `--meta` compares metadata by streaming both §5 +named sections in a lockstep merge-join — each side's entry is parsed +against its own §4 reverse index, compared, and dropped. Raw §5 +windows are *not* comparable across files (metadata name references +are file-relative §4 indices), so the sweep compares parsed, +Address-valued `ConstantMeta`. Exit codes follow GNU diff: 0 = no +difference in the selected mode, 1 = differences, 2 = error. + --- ## Proofs and Claims From 5facb0cb1677f42fd5cc45f08f2232498a8e2ec6 Mon Sep 17 00:00:00 2001 From: "John C. Burnham" Date: Fri, 10 Jul 2026 02:19:05 -0400 Subject: [PATCH 7/8] Ixon: streaming-metadata ix pack + --anon bundles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pack no longer uses the full reader. The source env is memory-mapped and lazily loaded (constant windows stay zero-copy mmap slices), and display metadata is carried by re-streaming §5 with a NamedMetaCursor once per prune fixpoint round, materializing Named entries only for carried constants — resident metadata is O(survivors) instead of O(env). Packing Euclid out of the 3.2 GB mathlib env: 22 s / 5.3 GiB peak (was ~60 s at full-reader tens-of-GiB), output byte-identical to the full-reader bundle. Engine: prune_to_closure refactored into prune_init + prune_value_pass + carry_named_entry (the named-pass body, parameterized over name/blob resolvers) — prune_to_closure_streaming shares it verbatim, so the in-memory and streaming paths cannot drift; a byte-identity test (with an aux-original fixture forcing a second fixpoint round, a §4-resolved binder name, and a meta_refs blob) locks it. parse_lazy_index_with_names returns the §4 Address→Name lookup the walk already builds (the plain variant keeps dropping it). --anon packs only anonymous structure via prune_to_closure_anon: value closure + §3 hints, empty §4/§5 — the minimal artifact a receiver needs to typecheck/evaluate the pinned value (validate_closed checks the value pin only). Anon Euclid: 2.5 MB vs 13 MB (3,138 consts, 89 value blobs vs 9,504 with name strings), 13.6 s. FFI rs_pack_env: mmap + lazy load, gains the anon flag (arity 5→6); mmap_file shared with the diff module. CLI: ix pack --anon. Tests: streaming/full byte parity (+ assumed-cut variant), anon value-only contents + roundtrip, Lean FFI anon-bundle fixture; 249 ixon tests, workspace + ffi/ixon suites green. --- Ix/Cli/PackCmd.lean | 9 +- Ix/Ixon.lean | 21 +- Tests/FFI/Ixon.lean | 13 +- crates/ffi/src/lean_ixon/diff.rs | 6 +- crates/ffi/src/lean_ixon/pack.rs | 89 ++++---- crates/ixon/src/env.rs | 380 +++++++++++++++++++++++-------- crates/ixon/src/serialize.rs | 71 +++++- docs/Ixon.md | 22 +- 8 files changed, 457 insertions(+), 154 deletions(-) diff --git a/Ix/Cli/PackCmd.lean b/Ix/Cli/PackCmd.lean index bfad1b7cb..390a7fb87 100644 --- a/Ix/Cli/PackCmd.lean +++ b/Ix/Cli/PackCmd.lean @@ -42,11 +42,13 @@ def runPackCmd (p : Cli.Parsed) : IO UInt32 := do match p.flag? "out" with | some flag => flag.as! String | none => s!"{mainName}.ixe" - let verbose := (p.flag? "verbose").isSome + let anon := p.hasFlag "anon" + let verbose := p.hasFlag "verbose" try - Ixon.rsPackEnv envPath mainName assume outPath verbose + Ixon.rsPackEnv envPath mainName assume outPath anon verbose + let mode := if anon then " [anon]" else "" IO.println s!"[pack] wrote {outPath} (main {mainName}, \ - {assume.size} assumption cut(s) declared)" + {assume.size} assumption cut(s) declared){mode}" return (0 : UInt32) catch e => IO.eprintln s!"error: {e.toString}" @@ -60,6 +62,7 @@ def packCmd : Cli.Cmd := `[Cli| "Prune a `.ixe` env to the self-contained bundle pinning one constant (sets `main`; validated closed)" FLAGS: + anon; "Pack only anonymous structure — no names or metadata (§4/§5 empty; §3 hints still carried). The minimal artifact a receiver needs to typecheck/evaluate the pinned value." assume : String; "Comma-separated cut-point constants — displayed names or 64-hex addresses. Reached cut-points are recorded in the bundle's `assumptions` instead of carried (thin bundle)." "assume-file" : String; "Additionally read cut-points from a file (one per line; `#` comments and blank lines ignored). Unions with --assume." out : String; "Output `.ixe` path. Defaults to `.ixe` (e.g. `Nat.add.ixe`)." diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index fe96b66d2..626337ac8 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -2383,17 +2383,20 @@ opaque rsIxeFilesEqual : @& String → @& String → IO Bool /-! ### Env pack (`rs_pack_env`) -/ -/-- Pack a value bundle in Rust: read the env at `envPath` (full - reader), resolve `mainName` (displayed form) to its constant - address, prune to the self-contained closure - (`Env::prune_to_closure` — sets `main`, collects reached cut-points - into `assumptions`, carries display metadata to fixpoint), validate - (`Env::validate_closed`), and write the bundle to `outPath`. - `assume` entries resolve as displayed names first, else as 64-hex - constant addresses. Failures surface as `IO` errors. -/ +/-- Pack a value bundle in Rust: memory-map the env at `envPath` + (lazy reader — metadata is never bulk-materialized), resolve + `mainName` (displayed form) to its constant address, prune to the + self-contained closure — `main` set, reached cut-points recorded in + `assumptions`; display metadata carried to fixpoint by re-streaming + §5 per round, or skipped entirely when `anon` is true (value + closure + §3 hints only, the minimal typecheck/eval artifact) — + validate (`Env::validate_closed`), and write the bundle to + `outPath`. `assume` entries resolve as displayed names first, else + as 64-hex constant addresses. Failures surface as `IO` errors. + Arg order: envPath, mainName, assume, outPath, anon, verbose. -/ @[extern "rs_pack_env"] opaque rsPackEnv : @& String → @& String → @& Array String → @& String → - Bool → IO Unit + Bool → Bool → IO Unit /-! ## Canonical merkle root over consts -/ diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index 2086e0cb1..b0107c69d 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -420,14 +420,15 @@ population — and the closed-subset relationship to the source env. -/ /-- Write `env` to a temp `.ixe`, pack `mainName` with `assume` cuts, and parse the resulting bundle. Pack failures surface as `.error`. -/ private def packFixture (env : Env) (mainName : String) - (assume : Array String) : IO (Except String Env) := do + (assume : Array String) (anon : Bool := false) : + IO (Except String Env) := do let dir ← IO.FS.createTempDir let src := dir / "src.ixe" let out := dir / "bundle.ixe" -- Every failure mode is caught into `.error`, so cleanup always runs. let result ← try IO.FS.writeBinFile src (serEnv env) - Ixon.rsPackEnv src.toString mainName assume out.toString false + Ixon.rsPackEnv src.toString mainName assume out.toString anon false let bytes ← IO.FS.readBinFile out pure (Ixon.rsDeEnv bytes) catch e => @@ -521,6 +522,14 @@ def envPackTests : TestSeq := packTest "EnvPack: assume cut (by hex address) records assumption" (packFixture src "bar" #[toString fooAddr]) (fun b => b.consts.size == 1 && b.assumptions.contains fooAddr) ++ + packTest "EnvPack: anon bundle has no metadata, keeps the value pin" + (packFixture src "bar" #[] (anon := true)) (fun b => + b.main == some barAddr + && b.named.isEmpty + -- The writer always emits the anonymous name as §4 entry 0. + && b.names.size <= 1 + && b.consts.size == 2 + && (b.getAddr? barName).isNone) ++ packErrTest "EnvPack: unknown root name errors" (packFixture src "nope" #[]) ++ packErrTest "EnvPack: main cannot be assumed" diff --git a/crates/ffi/src/lean_ixon/diff.rs b/crates/ffi/src/lean_ixon/diff.rs index 36868ebec..6bbbc2e64 100644 --- a/crates/ffi/src/lean_ixon/diff.rs +++ b/crates/ffi/src/lean_ixon/diff.rs @@ -268,7 +268,11 @@ pub extern "C" fn rs_diff_envs( } /// Memory-map a file, guarding against a concurrent length change. -fn mmap_file(path: &str, which: &str) -> Result, String> { +/// Shared with `rs_pack_env` (`super::pack`). +pub(crate) fn mmap_file( + path: &str, + which: &str, +) -> Result, String> { let file = std::fs::File::open(path) .map_err(|e| format!("{which} input {path}: {e}"))?; let expected = diff --git a/crates/ffi/src/lean_ixon/pack.rs b/crates/ffi/src/lean_ixon/pack.rs index 00868ee7c..d71711034 100644 --- a/crates/ffi/src/lean_ixon/pack.rs +++ b/crates/ffi/src/lean_ixon/pack.rs @@ -1,15 +1,16 @@ //! `rs_pack_env`: read a serialized env, prune it to the self-contained -//! closure of one named constant ([`ixon::env::Env::prune_to_closure`] — -//! sets `main` and collects reached cut-points into `assumptions`), -//! validate the result ([`ixon::env::Env::validate_closed`]), and write -//! the bundle `.ixe`. +//! closure of one named constant ([`ixon::env::Env::prune_to_closure`] +//! semantics — sets `main` and collects reached cut-points into +//! `assumptions`), validate the result +//! ([`ixon::env::Env::validate_closed`]), and write the bundle `.ixe`. //! -//! Uses the full reader (`Env::get`): the prune's named fixpoint carries -//! display metadata (names, `DataValue` blobs, aux `original`s), which -//! the anon/lazy views drop. At mathlib scale that means tens of GB -//! resident for the source env — functional, but heavy; a leaner -//! selective-metadata pack (stream §5, carry only survivors) is future -//! work on the diff meta-sweep streaming machinery. +//! The source env is memory-mapped and lazily loaded (constant windows +//! stay zero-copy mmap slices); metadata is never bulk-materialized. +//! The default mode carries display metadata by re-streaming §5 per +//! prune fixpoint round (`Env::prune_to_closure_streaming` — +//! O(survivors) resident metadata); `anon` mode skips metadata +//! entirely (`Env::prune_to_closure_anon` — value closure + §3 hints +//! only, the minimal typecheck/eval artifact). use ix_common::address::Address; use ixon::env::Env as IxonEnv; @@ -18,9 +19,11 @@ use lean_ffi::object::{ }; use rustc_hash::{FxHashMap, FxHashSet}; +use super::diff::mmap_file; + /// FFI: pack a value bundle. /// `(envPath mainName : String) → (assume : Array String) → -/// (outPath : String) → (verbose : Bool) → IO Unit`. +/// (outPath : String) → (anon verbose : Bool) → IO Unit`. /// /// `assume` entries resolve as displayed constant names first, else as /// 64-hex constant addresses (cut points need not be named). @@ -30,53 +33,55 @@ pub extern "C" fn rs_pack_env( main_name: LeanString>, assume: LeanArray>, out_path: LeanString>, + anon: LeanBool>, verbose: LeanBool>, ) -> LeanIOResult { + let anon = anon.to_bool(); let verbose = verbose.to_bool(); let path = env_path.to_string(); let main_str = main_name.to_string(); let out = out_path.to_string(); let assume_vec: Vec = assume.map(|obj| obj.as_string().to_string()); - let bytes = match std::fs::read(&path) { - Ok(bytes) => bytes, - Err(e) => { - return LeanIOResult::error_string(&format!( - "rs_pack_env: failed to read {path}: {e}" - )); - }, + let mmap = match mmap_file(&path, "source") { + Ok(m) => m, + Err(e) => return LeanIOResult::error_string(&format!("rs_pack_env: {e}")), }; if verbose { eprintln!( - "[rs_pack_env] parsing {path} ({} MB, full reader)...", - bytes.len() / 1_000_000 + "[rs_pack_env] parsing {path} ({} MB, lazy reader)...", + mmap.len() / 1_000_000 ); } - let mut slice: &[u8] = &bytes; - let full = match IxonEnv::get(&mut slice) { + let (index, names) = match IxonEnv::parse_lazy_index_with_names(&mmap[..]) { + Ok(v) => v, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_pack_env: failed to index {path}: {e}" + )); + }, + }; + let src = match IxonEnv::from_lazy_index_mmap(&index, &mmap) { Ok(env) => env, Err(e) => { return LeanIOResult::error_string(&format!( - "rs_pack_env: failed to deserialize {path}: {e}" + "rs_pack_env: failed to load {path}: {e}" )); }, }; if verbose { eprintln!( "[rs_pack_env] source env: {} consts, {} named, {} blobs", - full.consts.len(), - full.named.len(), - full.blobs.len() + src.consts.len(), + src.named.len(), + src.blobs.len() ); } - // Resolve displayed names → addresses through the full env's `named` - // metadata (the `rs_env_extract` idiom). - let by_name: FxHashMap = full - .named - .iter() - .map(|e| (e.key().to_string(), e.value().addr.clone())) - .collect(); + // Resolve displayed names → addresses through the lazy index's + // name→addr entries (the `rs_env_extract` idiom). + let by_name: FxHashMap = + index.named.iter().map(|n| (n.name.to_string(), n.addr.clone())).collect(); let main = match by_name.get(&main_str) { Some(a) => a.clone(), None => { @@ -104,7 +109,12 @@ pub extern "C" fn rs_pack_env( )); } - let bundle = match full.prune_to_closure(&main, &assumed) { + let bundle = if anon { + src.prune_to_closure_anon(&main, &assumed) + } else { + src.prune_to_closure_streaming(&index, &mmap[..], &names, &main, &assumed) + }; + let bundle = match bundle { Ok(b) => b, Err(e) => return LeanIOResult::error_string(&format!("rs_pack_env: {e}")), }; @@ -126,14 +136,15 @@ pub extern "C" fn rs_pack_env( eprintln!("[rs_pack_env] main {} ({main_str})", main.hex()); eprintln!( "[rs_pack_env] kept {}/{} consts, {}/{} named, {}/{} blobs, \ - {} assumption(s)", + {} assumption(s){}", bundle.consts.len(), - full.consts.len(), + src.consts.len(), bundle.named.len(), - full.named.len(), + src.named.len(), bundle.blobs.len(), - full.blobs.len(), - bundle.assumptions.len() + src.blobs.len(), + bundle.assumptions.len(), + if anon { " [anon: no display metadata]" } else { "" } ); eprintln!("[rs_pack_env] wrote {out} ({} bytes)", buf.len()); } diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index 0043b9333..c4a036a65 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -725,106 +725,29 @@ impl Env { main: &Address, assumed: &FxHashSet
, ) -> Result { - if assumed.contains(main) { - return Err("prune_to_closure: main cannot be assumed".to_string()); - } - let mut out = Env::new(); - out.main = Some(main.clone()); - - let mut visited: FxHashSet
= FxHashSet::default(); - let mut pending: VecDeque
= VecDeque::new(); - visited.insert(main.clone()); - pending.push_back(main.clone()); + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; let mut named_done: FxHashSet = FxHashSet::default(); - loop { - // ── Value pass: 3-edge BFS over pending roots, cut at `assumed`. - while let Some(addr) = pending.pop_front() { - if assumed.contains(&addr) { - out.assumptions.insert(addr); - continue; - } - if let Some(bytes) = self.get_const_bytes(&addr) { - out.store_const_lazy(addr.clone(), bytes); - if let Some(h) = self.anon_hints.get(&addr) { - out.anon_hints.insert(addr.clone(), *h); - } - let c = self.get_const(&addr).ok_or_else(|| { - format!("prune_to_closure: constant {} unparseable", addr.hex()) - })?; - let mut edges: Vec
= Vec::new(); - Self::closure_edges(&addr, &c, &mut edges); - for e in edges { - if visited.insert(e.clone()) { - pending.push_back(e); - } - } - } else if let Some(blob) = self.get_blob(&addr) { - out.blobs.insert(addr, blob); - } else { - return Err(format!( - "prune_to_closure: {} reachable from main but not in \ - consts/blobs and not assumed", - addr.hex() - )); - } - } + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; // ── Named pass: carry display metadata for every carried // constant. Metadata references content the value walk cannot // see; new DAG edges feed the next value pass. for entry in self.named.iter() { - let named = entry.value(); - if !out.consts.contains_key(&named.addr) - || named_done.contains(entry.key()) - { + let (name, named) = (entry.key(), entry.value()); + if !out.consts.contains_key(&named.addr) || named_done.contains(name) { continue; } - named_done.insert(entry.key().clone()); - out.named.insert(entry.key().clone(), named.clone()); - self.carry_name(&mut out, entry.key()); - - let mut name_addrs: Vec
= Vec::new(); - let mut blob_addrs: Vec
= Vec::new(); - let mut dag_addrs: Vec
= Vec::new(); - named.meta().collect_deps( - &mut name_addrs, - &mut blob_addrs, - &mut dag_addrs, - ); - if let Some((orig_addr, orig_meta)) = named.original() { - dag_addrs.push(orig_addr); - orig_meta.collect_deps( - &mut name_addrs, - &mut blob_addrs, - &mut dag_addrs, - ); - } - for na in name_addrs { - let Some(name) = self.get_name(&na) else { - return Err(format!( - "prune_to_closure: metadata references name {} absent from \ - names", - na.hex() - )); - }; - self.carry_name(&mut out, &name); - } - for ba in blob_addrs { - let Some(blob) = self.get_blob(&ba) else { - return Err(format!( - "prune_to_closure: metadata references blob {} absent from \ - blobs", - ba.hex() - )); - }; - out.blobs.insert(ba, blob); - } - for da in dag_addrs { - if visited.insert(da.clone()) { - pending.push_back(da); - } - } + named_done.insert(name.clone()); + Self::carry_named_entry( + &mut out, + name, + named, + &|na| self.get_name(na), + &|ba| self.get_blob(ba), + &mut visited, + &mut pending, + )?; } // The named pass ran against the final consts of this round; if @@ -836,13 +759,151 @@ impl Env { Ok(out) } + /// Value-only bundle: the 3-edge closure of `main` cut at `assumed` — + /// constants (genuine bytes), value blobs, per-constant hints, + /// `main`, and reached assumptions. No display metadata at all: + /// `names`/`named` stay empty (§4/§5 serialize as empty sections). + /// The result still passes [`Self::validate_closed`] (which checks + /// the value pin only) — this is the minimal artifact a receiver + /// needs to typecheck/evaluate the pinned value. Host-only — see + /// `store_blob`. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure_anon( + &self, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; + Ok(out) + } + + /// Shared prune setup: assumed-main guard, `out` with `main` set, + /// seeded worklist. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn prune_init( + main: &Address, + assumed: &FxHashSet
, + ) -> Result<(Env, FxHashSet
, VecDeque
), String> { + if assumed.contains(main) { + return Err("prune_to_closure: main cannot be assumed".to_string()); + } + let mut out = Env::new(); + out.main = Some(main.clone()); + let mut visited: FxHashSet
= FxHashSet::default(); + let mut pending: VecDeque
= VecDeque::new(); + visited.insert(main.clone()); + pending.push_back(main.clone()); + Ok((out, visited, pending)) + } + + /// One value pass: 3-edge BFS over pending roots, cut at `assumed`. + /// Carries constant bytes + per-constant hints + reached blobs; + /// records reached cut points in `out.assumptions`. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn prune_value_pass( + &self, + out: &mut Env, + visited: &mut FxHashSet
, + pending: &mut VecDeque
, + assumed: &FxHashSet
, + ) -> Result<(), String> { + while let Some(addr) = pending.pop_front() { + if assumed.contains(&addr) { + out.assumptions.insert(addr); + continue; + } + if let Some(bytes) = self.get_const_bytes(&addr) { + out.store_const_lazy(addr.clone(), bytes); + if let Some(h) = self.anon_hints.get(&addr) { + out.anon_hints.insert(addr.clone(), *h); + } + let c = self.get_const(&addr).ok_or_else(|| { + format!("prune_to_closure: constant {} unparseable", addr.hex()) + })?; + let mut edges: Vec
= Vec::new(); + Self::closure_edges(&addr, &c, &mut edges); + for e in edges { + if visited.insert(e.clone()) { + pending.push_back(e); + } + } + } else if let Some(blob) = self.get_blob(&addr) { + out.blobs.insert(addr, blob); + } else { + return Err(format!( + "prune_to_closure: {} reachable from main but not in \ + consts/blobs and not assumed", + addr.hex() + )); + } + } + Ok(()) + } + + /// Carry one named entry and its metadata dependencies into `out`: + /// the `Named` row, its name's component chain, every + /// metadata-referenced name component and blob, and (via + /// `visited`/`pending`) any new constant DAG edges (aux `original`s, + /// `meta_refs`) for the next value pass. `resolve_name`/`get_blob` + /// abstract the source: the in-memory env for + /// [`Self::prune_to_closure`]; the §4 lookup + lazy env for the + /// streaming variant (`Env::prune_to_closure_streaming` in + /// `serialize.rs`) — one body, so the two paths cannot drift. + #[cfg(not(target_arch = "riscv64"))] + pub(crate) fn carry_named_entry( + out: &mut Env, + name: &Name, + named: &Named, + resolve_name: &dyn Fn(&Address) -> Option, + get_blob: &dyn Fn(&Address) -> Option>, + visited: &mut FxHashSet
, + pending: &mut VecDeque
, + ) -> Result<(), String> { + out.named.insert(name.clone(), named.clone()); + Self::carry_name(out, name); + + let mut name_addrs: Vec
= Vec::new(); + let mut blob_addrs: Vec
= Vec::new(); + let mut dag_addrs: Vec
= Vec::new(); + named.meta().collect_deps(&mut name_addrs, &mut blob_addrs, &mut dag_addrs); + if let Some((orig_addr, orig_meta)) = named.original() { + dag_addrs.push(orig_addr); + orig_meta.collect_deps(&mut name_addrs, &mut blob_addrs, &mut dag_addrs); + } + for na in name_addrs { + let Some(name) = resolve_name(&na) else { + return Err(format!( + "prune_to_closure: metadata references name {} absent from names", + na.hex() + )); + }; + Self::carry_name(out, &name); + } + for ba in blob_addrs { + let Some(blob) = get_blob(&ba) else { + return Err(format!( + "prune_to_closure: metadata references blob {} absent from blobs", + ba.hex() + )); + }; + out.blobs.insert(ba, blob); + } + for da in dag_addrs { + if visited.insert(da.clone()) { + pending.push_back(da); + } + } + Ok(()) + } + /// Copy `name` and its full parent chain into `out.names`, storing /// each string component's bytes as a blob (the compiler's /// convention — mirrors `addNameComponentsWithBlobs` on the Lean /// side). Stops early once a component is already present: its /// parents were carried with it. #[cfg(not(target_arch = "riscv64"))] - fn carry_name(&self, out: &mut Env, name: &Name) { + fn carry_name(out: &mut Env, name: &Name) { use ix_common::env::NameData; let mut cur = name.clone(); loop { @@ -1468,4 +1529,139 @@ mod tests { let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); assert_eq!(anon.anon_hints.get(&a), Some(&ReducibilityHints::Regular(3))); } + + /// Fixture for the streaming/anon prune tests: `a` (the main) named + /// "Foo" with Def metadata carrying (i) a §4-resolved non-ancestor + /// name component (`u`, in `lvls`), (ii) a `meta_refs` blob, and + /// (iii) an `original` edge to `b` — a constant NOT reachable from + /// `a`'s value, so carrying it requires a second fixpoint round; `b` + /// is named "Bar". Returns (serialized env, a, b, "Bar"). + fn streaming_prune_fixture() -> (Vec, Address, Address, Name) { + use crate::metadata::{ConstantMetaInfo, ExprMeta}; + let env = Env::new(); + let a = store_canonical(&env, const_with_refs(vec![])); + let b = store_canonical(&env, const_with_refs(vec![a.clone()])); + + let foo = n("Foo"); + let foo_addr = Address::from_blake3_hash(*foo.get_hash()); + env.store_name(foo_addr.clone(), foo.clone()); + let u = n("u"); + let u_addr = Address::from_blake3_hash(*u.get_hash()); + env.store_name(u_addr.clone(), u.clone()); + let meta_blob = env.store_blob(b"payload".to_vec()); + let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { + name: foo_addr, + lvls: vec![u_addr], + hints: ReducibilityHints::Regular(2), + all: vec![], + ctx: vec![], + arena: ExprMeta::default(), + type_root: 0, + value_root: 0, + }); + meta.meta_refs.push(meta_blob); + let mut foo_named = Named::new(a.clone(), meta); + foo_named.set_original(b.clone(), ConstantMeta::default()); + env.register_name(foo, foo_named); + let bar = n("Bar"); + let bar_addr = Address::from_blake3_hash(*bar.get_hash()); + env.store_name(bar_addr, bar.clone()); + env.register_name(bar.clone(), Named::with_addr(b.clone())); + + let mut bytes = Vec::new(); + env.put(&mut bytes).unwrap(); + (bytes, a, b, bar) + } + + /// The streaming prune (lazy env + §5 re-stream per fixpoint round) + /// must produce a byte-identical bundle to the in-memory prune over + /// the full reader — including metadata that forces a second round. + #[test] + fn prune_streaming_matches_full_byte_identical() { + let (bytes, a, b, bar) = streaming_prune_fixture(); + let full = { + let mut cur = bytes.as_slice(); + Env::get(&mut cur).unwrap() + }; + let (index, names) = Env::parse_lazy_index_with_names(&bytes).unwrap(); + let lazy = Env::from_lazy_index(&index, &bytes).unwrap(); + let ser = |e: &Env| { + let mut v = Vec::new(); + e.put(&mut v).unwrap(); + v + }; + + let via_full = full.prune_to_closure(&a, &FxHashSet::default()).unwrap(); + let via_stream = lazy + .prune_to_closure_streaming( + &index, + &bytes, + &names, + &a, + &FxHashSet::default(), + ) + .unwrap(); + // Round-2 evidence: Bar rides in through Foo's `original` edge. + assert!(via_stream.named.get(&bar).is_some(), "round-2 named carried"); + assert!(via_stream.consts.contains_key(&b)); + via_stream.validate_closed().unwrap(); + assert_eq!( + ser(&via_full), + ser(&via_stream), + "streaming bundle must be byte-identical to the full-reader bundle" + ); + + // With `b` assumed: the original edge lands in assumptions, Bar is + // not carried — parity still holds. + let assumed: FxHashSet
= [b.clone()].into_iter().collect(); + let via_full_cut = full.prune_to_closure(&a, &assumed).unwrap(); + let via_stream_cut = lazy + .prune_to_closure_streaming(&index, &bytes, &names, &a, &assumed) + .unwrap(); + assert!(via_stream_cut.named.get(&bar).is_none()); + assert!(via_stream_cut.assumptions.contains(&b)); + assert_eq!(ser(&via_full_cut), ser(&via_stream_cut)); + } + + /// `prune_to_closure_anon` carries the value closure only: no names, + /// no named entries, no metadata-edge constants or blobs — the + /// minimal typecheck/eval artifact. Hints (§3) still ride; the + /// bundle validates closed and round-trips with empty §4/§5. + #[test] + fn prune_anon_value_only() { + let (bytes, a, b, _) = streaming_prune_fixture(); + let (index, _) = Env::parse_lazy_index_with_names(&bytes).unwrap(); + let lazy = Env::from_lazy_index(&index, &bytes).unwrap(); + + let bundle = lazy.prune_to_closure_anon(&a, &FxHashSet::default()).unwrap(); + assert!( + bundle.named.is_empty(), + "no named entries: {}", + bundle.named.len() + ); + assert!(bundle.names.is_empty(), "no name components"); + assert!(bundle.blobs.is_empty(), "no metadata/name-string blobs"); + assert!(bundle.consts.contains_key(&a)); + assert!( + !bundle.consts.contains_key(&b), + "metadata-only edges must not be walked in anon mode" + ); + // §3 hints derive from the source's Named Def metadata at write + // time and ride the lazy env's anon_hints into the bundle. + assert_eq!(bundle.anon_hints.get(&a), Some(&ReducibilityHints::Regular(2))); + bundle.validate_closed().unwrap(); + + let mut buf = Vec::new(); + bundle.put(&mut buf).unwrap(); + let back = { + let mut cur = buf.as_slice(); + Env::get(&mut cur).unwrap() + }; + assert!(back.named.is_empty(), "no §5 entries after roundtrip"); + // The writer always emits the anonymous name as §4 entry 0. + assert!(back.names.len() <= 1, "§4 carries at most the anon entry"); + assert_eq!(back.main, Some(a.clone())); + let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); + assert_eq!(anon.anon_hints.get(&a), Some(&ReducibilityHints::Regular(2))); + } } diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index cb41fc2d2..436574f13 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -999,6 +999,9 @@ impl Constant { use bignat::Nat; use ix_common::env::{Name, NameData}; use rustc_hash::FxHashMap; +// Used only by the host-gated streaming prune's signature. +#[cfg(not(target_arch = "riscv64"))] +use rustc_hash::FxHashSet; /// Serialize a Name to bytes (full recursive serialization, for standalone use). pub fn put_name(name: &Name, buf: &mut Vec) { @@ -1912,6 +1915,18 @@ impl Env { /// this reader now consumes every section, trailing bytes are rejected /// exactly as in [`Env::get`]. pub fn parse_lazy_index(data: &[u8]) -> Result { + Self::parse_lazy_index_with_names(data).map(|(index, _)| index) + } + + /// [`Env::parse_lazy_index`], additionally returning the §4 + /// Address→Name component lookup. The lookup is built during the + /// walk regardless (§4 parsing resolves parent references through + /// it); the plain variant just drops it. Consumers that resolve + /// metadata name references from §5 windows — the streaming prune — + /// need it. + pub fn parse_lazy_index_with_names( + data: &[u8], + ) -> Result<(LazyIndex, FxHashMap), String> { let mut buf: &[u8] = data; // Header: tag + merkle root + bundle fields. @@ -2027,7 +2042,61 @@ impl Env { )); } - Ok(index) + Ok((index, names_lookup)) + } + + /// [`Env::prune_to_closure`] over a lazily-loaded env — the + /// streaming-metadata pack path. `self` must be the env built by + /// [`Env::from_lazy_index`]/[`Env::from_lazy_index_mmap`] from + /// `index` over `data`, and `names` the §4 lookup from + /// [`Env::parse_lazy_index_with_names`]. Each fixpoint round + /// re-streams §5 with a [`NamedMetaCursor`], materializing `Named` + /// entries only for carried constants — resident metadata is + /// O(survivors) instead of O(env), the full reader's cost. The carry + /// logic is [`Env::carry_named_entry`], shared with the in-memory + /// prune, so the two paths produce identical bundles. + #[cfg(not(target_arch = "riscv64"))] + pub fn prune_to_closure_streaming( + &self, + index: &LazyIndex, + data: &[u8], + names: &FxHashMap, + main: &Address, + assumed: &FxHashSet
, + ) -> Result { + let (mut out, mut visited, mut pending) = Self::prune_init(main, assumed)?; + let mut named_done: FxHashSet = FxHashSet::default(); + loop { + self.prune_value_pass(&mut out, &mut visited, &mut pending, assumed)?; + + // ── Named pass, streamed: every §5 entry is parsed (boundaries + // require it — no length sidecar), but only entries whose + // constant was carried this round materialize into `out`. + let mut cursor = NamedMetaCursor::open(data, index)?; + let mut i = 0usize; + while let Some((_, named)) = cursor.next_entry()? { + let name = &index.named[i].name; + i += 1; + if !out.consts.contains_key(&named.addr) || named_done.contains(name) { + continue; + } + named_done.insert(name.clone()); + Self::carry_named_entry( + &mut out, + name, + &named, + &|na| names.get(na).cloned(), + &|ba| self.get_blob(ba), + &mut visited, + &mut pending, + )?; + } + + if pending.is_empty() { + break; + } + } + Ok(out) } /// Anonymous-only deserialization: read the header + §1 blobs + diff --git a/docs/Ixon.md b/docs/Ixon.md index e6dbe66ad..52cedb067 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -938,13 +938,21 @@ lake exe ix pack [--out ] ``` It resolves `` (displayed form) against the env's `named` table, -runs `prune_to_closure`, re-validates with `validate_closed`, and -writes the bundle (default `.ixe`). `--assume` entries — names or -64-hex constant addresses — declare trust-boundary cut points; the ones -actually reached become the bundle's `assumptions` (thin bundle). Pack -reads with the full reader so display metadata survives the prune; -compile once, pack many. (`ix shard extract` is the non-bundle sibling: -a general sub-env for the kernel-check pipeline, no `main` root.) +runs the prune, re-validates with `validate_closed`, and writes the +bundle (default `.ixe`). `--assume` entries — names or 64-hex +constant addresses — declare trust-boundary cut points; the ones +actually reached become the bundle's `assumptions` (thin bundle). + +The source env is memory-mapped and lazily loaded; display metadata is +carried by **re-streaming §5 per prune fixpoint round** +(`Env::prune_to_closure_streaming`) so resident metadata is +O(survivors), not O(env) — byte-identical output to the in-memory +`prune_to_closure`, which shares the same carry engine. `--anon` skips +metadata entirely (`Env::prune_to_closure_anon`): value closure + §3 +hints, empty §4/§5 — the minimal artifact a receiver needs to +typecheck/evaluate the pinned value, since `validate_closed` checks +only the value pin. (`ix shard extract` is the non-bundle sibling: a +general sub-env for the kernel-check pipeline, no `main` root.) ### Diffing environments From 250277dc25eed393826b1bd544a36ab954b41e50 Mon Sep 17 00:00:00 2001 From: Samuel Burnham <45365069+samuelburnham@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:13:58 -0400 Subject: [PATCH 8/8] refactor: single-source reducibility hints in Env::anon_hints (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the hints field from ConstantMeta::Def (Rust, Lean, the FFI constructor slots, and the .ixe named-section encoding). Reducibility hints now live only in the env-level anon_hints map, keyed by constant address. - The compiler is the sole producer: compile_definition records each definition's hints per name (CompileState::def_hints), and finalize_hints resolves them through the registered Named entries into env.anon_hints via Env::register_hint once addresses are final. The Lean compiler mirror (BlockState.defHints + Ixon.mergeHints) does the same. Alias collisions merge order-independently (min by (tag, height)), so parallel workers and both mirrors agree byte-for-byte. - Writers serialize the map directly; the derive-from-Named fallback is deleted from Env::put / put_file / serialized_size_breakdown and both Lean writers. This removes the serial full-decode of every demoted Named entry that regressed ix compile by 5-14%: the derivation loop called Named::meta() per entry, which re-decodes demoted metadata (the IX_COMPILE_DEMOTE default) on every call. - Kernel ingress reads hints from anon_hints in both anon and meta modes; the hints_override parameter threading is gone. Decompile (both languages) and the IxVM claim harness look hints up by address. The lazy check path transports the hints section on RawEnvLazy instead of fabricating per-name Def metas (RawNamedLite.toConstMeta is removed). - anon_hints becomes an IxonMap like the other env maps; Env now derives Clone (the riscv64 IxonMap wrapper gained Clone, replacing the manual per-map Clone impl). Format change (pre-alpha, no compat shims): Def metadata loses its hints byte in the named sections, so every hint is stored once instead of two or three times (hints section + named meta + aux originals) — the whole-stdlib env shrinks by ~232 KB. InitStd compile-time returns to its pre-hints-section cost (median 7.3s -> 5.7s locally vs the derivation code), with byte-exact Lean/Rust writer parity at 505,674,085 bytes on the whole-stdlib env. Behavior note: a hint-only change now reports as a pure hintsChanged row in ix diff; it previously also surfaced as a Def metadata diff. --- Ix/CompileM.lean | 50 ++++- Ix/DecompileM.lean | 19 +- Ix/IxVM/ClaimHarness.lean | 11 +- Ix/Ixon.lean | 109 ++++------ Tests/FFI/Ixon.lean | 22 ++- Tests/Gen/Ixon.lean | 2 +- Tests/Ix/Compile.lean | 5 +- crates/compile/src/compile.rs | 30 ++- crates/compile/src/compile/env.rs | 2 + crates/compile/src/decompile.rs | 20 +- crates/ffi/src/lean.rs | 10 +- crates/ffi/src/lean_ixon/env.rs | 38 ++-- crates/ffi/src/lean_ixon/meta.rs | 24 +-- crates/ffi/src/lean_ixon/serialize.rs | 26 +-- crates/ixon/src/diff.rs | 29 +-- crates/ixon/src/env.rs | 143 +++++++------- crates/ixon/src/map.rs | 2 +- crates/ixon/src/metadata.rs | 15 +- crates/ixon/src/serialize.rs | 198 +++++++------------ crates/ixvm-codegen/src/aiur_ixvm_witness.rs | 2 +- crates/kernel/src/anon_work.rs | 8 +- crates/kernel/src/ingress.rs | 67 +++---- docs/Ixon.md | 21 +- 23 files changed, 398 insertions(+), 455 deletions(-) diff --git a/Ix/CompileM.lean b/Ix/CompileM.lean index 701608a55..a39fe51fd 100644 --- a/Ix/CompileM.lean +++ b/Ix/CompileM.lean @@ -84,6 +84,10 @@ structure BlockState where blockBlobs : Std.HashMap Address ByteArray := {} /-- Name components collected during block compilation -/ blockNames : Std.HashMap Address Ix.Name := {} + /-- Reducibility hints per definition name compiled in this block. + Hints are not part of `ConstantMeta`; the driver resolves this + map into `Ixon.Env.anonHints` once addresses are final. -/ + defHints : Std.HashMap Name Lean.ReducibilityHints := {} /-- Arena-based expression metadata for the current constant -/ arena : Ixon.ExprMetaArena := {} deriving Inhabited @@ -310,6 +314,10 @@ def storeString (s : String) : CompileM Address := do modifyBlockState fun c => { c with blockBlobs := c.blockBlobs.insert addr bytes } pure addr +/-- Record a definition's reducibility hints (see `BlockState.defHints`). -/ +def recordDefHints (name : Name) (hints : Lean.ReducibilityHints) : CompileM Unit := + modifyBlockState fun c => { c with defHints := c.defHints.insert name hints } + /-- Compile a name: store all string components as blobs and track name components in blockNames for deduplication. This matches Rust's compile_name behavior. -/ @@ -889,7 +897,8 @@ def compileDefinition (d : DefinitionVal) : CompileM (Ixon.Definition × Ixon.Co typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs d.hints allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name d.hints + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile a theorem to Ixon.Definition with metadata. -/ @@ -919,7 +928,8 @@ def compileTheorem (d : TheoremVal) : CompileM (Ixon.Definition × Ixon.Constant typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs .opaque allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name .opaque + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile an opaque to Ixon.Definition with metadata. -/ @@ -949,7 +959,8 @@ def compileOpaque (d : OpaqueVal) : CompileM (Ixon.Definition × Ixon.ConstantMe typ := typeExpr value := valueExpr } - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs .opaque allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.cnst.name .opaque + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile an axiom to Ixon.Axiom with metadata. -/ @@ -1155,7 +1166,8 @@ def compileDefinitionData (d : Def) : CompileM (Ixon.Definition × Ixon.Constant | .defn => d.hints | .thm => .opaque | .opaq => .opaque - let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs hints allAddrs ctxAddrs arena typeRoot valueRoot + recordDefHints d.name hints + let constMeta := Ixon.ConstantMeta.defn nameAddr lvlAddrs allAddrs ctxAddrs arena typeRoot valueRoot pure (defn, constMeta, typeExpr, valueExpr) /-- Compile inductive data for an Ind structure (from Mutual.lean). @@ -1510,6 +1522,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool -- Initialize compilation state let mut compileEnv := CompileEnv.new env let mut blockNames : Std.HashMap Address Ix.Name := {} + let mut defHints : Std.HashMap Name Lean.ReducibilityHints := {} -- Build work queue data structures let totalBlocks := blocks.blocks.size @@ -1554,6 +1567,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool blobs := cache.blockBlobs.fold (fun m k v => m.insert k v) compileEnv.blobs } blockNames := cache.blockNames.fold (fun m k v => m.insert k v) blockNames + defHints := cache.defHints.fold (fun m k v => m.insert k v) defHints -- If there are projections, store them and map names to projection addresses if result.projections.isEmpty then @@ -1608,6 +1622,17 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool -- Merge name string blobs into the main blobs map let allBlobs := nameBlobs.fold (fun m k v => m.insert k v) compileEnv.blobs + -- Resolve per-name hints to each name's registered constant address + -- (the projection address for mutual-block members — exactly the + -- address the kernel looks hints up under). Alias collisions merge + -- order-independently, matching Rust `CompileState::finalize_hints`. + let anonHints := compileEnv.nameToNamed.fold (init := {}) fun m name named => + match defHints.get? name with + | some h => m.alter named.addr fun + | some h₀ => some (Ixon.mergeHints h₀ h) + | none => some h + | none => m + let ixonEnv : Ixon.Env := { consts := compileEnv.constants.fold (init := {}) fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c) @@ -1616,6 +1641,7 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool names := namesMap comms := {} addrToName := addrToNameMap + anonHints } return .ok (ixonEnv, compileEnv.totalBytes) @@ -1705,6 +1731,7 @@ structure WaveBlockResult where projections : Array (Name × Ixon.Constant × Address × Ixon.ConstantMeta) blobs : Std.HashMap Address ByteArray names : Std.HashMap Address Ix.Name + defHints : Std.HashMap Name Lean.ReducibilityHints totalBytes : Nat /-- Work item for a worker thread -/ @@ -1793,6 +1820,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) projections := projsNoBytes blobs := cache.blockBlobs names := cache.blockNames + defHints := cache.defHints totalBytes := projBytes } discard <| resultChan.send result @@ -1808,6 +1836,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) let mut constants : Std.HashMap Address Ixon.Constant := {} let mut blobs : Std.HashMap Address ByteArray := {} let mut blockNames : Std.HashMap Address Ix.Name := {} + let mut defHints : Std.HashMap Name Lean.ReducibilityHints := {} let mut totalBytes : Nat := 0 let mut remaining : Set Name := {} @@ -1864,9 +1893,10 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) for (name, proj, addr, constMeta) in result.projections do constants := constants.insert addr proj nameToNamed := nameToNamed.insert name { addr, constMeta } - -- Store blobs and names + -- Store blobs, names, and hints blobs := result.blobs.fold (fun m k v => m.insert k v) blobs blockNames := result.names.fold (fun m k v => m.insert k v) blockNames + defHints := result.defHints.fold (fun m k v => m.insert k v) defHints totalBytes := totalBytes + result.totalBytes compiled := compiled + 1 @@ -1902,6 +1932,15 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) if dbg then IO.println s!" [Lean Compile] Blobs: {blockBlobCount} from blocks, {nameBlobCount} from names, {overlapCount} overlap, {finalBlobCount} final" + -- Resolve per-name hints to registered constant addresses (see the + -- serial driver / Rust `CompileState::finalize_hints`). + let anonHints := nameToNamed.fold (init := {}) fun m name named => + match defHints.get? name with + | some h => m.alter named.addr fun + | some h₀ => some (Ixon.mergeHints h₀ h) + | none => some h + | none => m + let ixonEnv : Ixon.Env := { consts := constants.fold (init := {}) fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c) @@ -1910,6 +1949,7 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks) names := namesMap comms := {} addrToName := addrToNameMap + anonHints } return .ok (ixonEnv, totalBytes) diff --git a/Ix/DecompileM.lean b/Ix/DecompileM.lean index 81ba6c133..e2f548a93 100644 --- a/Ix/DecompileM.lean +++ b/Ix/DecompileM.lean @@ -507,7 +507,7 @@ def getLvlAddrs : ConstantMeta → Array Address | .empty | .muts _ => #[] def getArenaAndTypeRoot : ConstantMeta → ExprMetaArena × UInt64 - | .defn _ _ _ _ _ arena typeRoot _ => (arena, typeRoot) + | .defn _ _ _ _ arena typeRoot _ => (arena, typeRoot) | .axio _ _ arena typeRoot => (arena, typeRoot) | .quot _ _ arena typeRoot => (arena, typeRoot) | .indc _ _ _ _ _ arena typeRoot => (arena, typeRoot) @@ -516,11 +516,11 @@ def getArenaAndTypeRoot : ConstantMeta → ExprMetaArena × UInt64 | .empty | .muts _ => ({}, 0) def getAllAddrs : ConstantMeta → Array Address - | .defn _ _ _ all .. => all | .indc _ _ _ all .. => all + | .defn _ _ all .. => all | .indc _ _ _ all .. => all | .recr _ _ _ all .. => all | _ => #[] def getCtxAddrs : ConstantMeta → Array Address - | .defn _ _ _ _ ctx .. => ctx | .indc _ _ _ _ ctx .. => ctx + | .defn _ _ _ ctx .. => ctx | .indc _ _ _ _ ctx .. => ctx | .recr _ _ _ _ ctx .. => ctx | _ => #[] /-- Resolve name from ConstantMeta. -/ @@ -571,9 +571,16 @@ def decompileDefinition (d : Ixon.Definition) (cnst : Constant) (cMeta : Constan let univParams ← decompileMetaLevels cMeta let allNames ← decompileMetaAll cMeta name let mutCtx ← decompileMetaCtx cMeta - let (hints, valueRoot) := match cMeta with - | .defn _ _ hints _ _ _ _ valueRoot => (hints, valueRoot) - | _ => (.opaque, (0 : UInt64)) + let valueRoot := match cMeta with + | .defn _ _ _ _ _ _ valueRoot => valueRoot + | _ => (0 : UInt64) + -- Hints live in `Env.anonHints`, keyed by the constant address the + -- name resolves to; absent entry → `.opaque`, matching the + -- compiler's treatment of theorems and opaques. + let ixonEnv := (← getEnv).ixonEnv + let hints := match ixonEnv.named.get? name with + | some named => (ixonEnv.anonHints.get? named.addr).getD .opaque + | none => .opaque let (arena, typeRoot) := getArenaAndTypeRoot cMeta withFreshBlock cnst mutCtx univParams arena do let typeExpr ← decompileExpr d.typ typeRoot diff --git a/Ix/IxVM/ClaimHarness.lean b/Ix/IxVM/ClaimHarness.lean index 55e29ce17..52cb70c4f 100644 --- a/Ix/IxVM/ClaimHarness.lean +++ b/Ix/IxVM/ClaimHarness.lean @@ -181,13 +181,10 @@ def addEntries (ixonEnv : Ixon.Env) (keep : Address → Bool) ioBuffer := ioBuffer.extend 5 key (rawBytes.data.map fun b => .ofNat b.toNat) -- Discriminator: this addr resolves to a blob. ioBuffer := ioBuffer.extend 4 key #[.ofNat 0] - for (_, named) in ixonEnv.named do - if !keep named.addr then continue - match named.constMeta with - | .defn _ _ hints _ _ _ _ _ => - let key : Array Aiur.G := named.addr.hash.data.map .ofUInt8 - ioBuffer := ioBuffer.extend 3 key #[hintToG hints] - | _ => pure () + for (addr, hints) in ixonEnv.anonHints do + if !keep addr then continue + let key : Array Aiur.G := addr.hash.data.map .ofUInt8 + ioBuffer := ioBuffer.extend 3 key #[hintToG hints] return ioBuffer -- ============================================================================ diff --git a/Ix/Ixon.lean b/Ix/Ixon.lean index 626337ac8..1be0476ee 100644 --- a/Ix/Ixon.lean +++ b/Ix/Ixon.lean @@ -475,7 +475,7 @@ def ExprMetaArena.mdataItemCount (arena : ExprMetaArena) : Nat := that constant, plus root indices pointing into the arena. -/ inductive ConstantMeta where | empty - | defn (name : Address) (lvls : Array Address) (hints : Lean.ReducibilityHints) + | defn (name : Address) (lvls : Array Address) (all : Array Address) (ctx : Array Address) (arena : ExprMetaArena) (typeRoot : UInt64) (valueRoot : UInt64) | axio (name : Address) (lvls : Array Address) @@ -497,7 +497,7 @@ inductive ConstantMeta where /-- Count total arena nodes in this ConstantMeta. -/ def ConstantMeta.exprMetaCount : ConstantMeta → Nat | .empty => 0 - | .defn _ _ _ _ _ arena _ _ => arena.nodes.size + | .defn _ _ _ _ arena _ _ => arena.nodes.size | .axio _ _ arena _ => arena.nodes.size | .quot _ _ arena _ => arena.nodes.size | .indc _ _ _ _ _ arena _ => arena.nodes.size @@ -508,7 +508,7 @@ def ConstantMeta.exprMetaCount : ConstantMeta → Nat /-- Count total arena nodes and mdata items in this ConstantMeta. -/ def ConstantMeta.exprMetaStats : ConstantMeta → Nat × Nat | .empty => (0, 0) - | .defn _ _ _ _ _ arena _ _ => (arena.nodes.size, arena.mdataItemCount) + | .defn _ _ _ _ arena _ _ => (arena.nodes.size, arena.mdataItemCount) | .axio _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) | .quot _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) | .indc _ _ _ _ _ arena _ => (arena.nodes.size, arena.mdataItemCount) @@ -522,7 +522,7 @@ def ConstantMeta.exprMetaByType : ConstantMeta → Nat × Nat × Nat × Nat × N | .empty => (0, 0, 0, 0, 0) | cm => let arena := match cm with - | .defn _ _ _ _ _ a _ _ => a + | .defn _ _ _ _ a _ _ => a | .axio _ _ a _ => a | .quot _ _ a _ => a | .indc _ _ _ _ _ a _ => a @@ -1181,6 +1181,22 @@ def getReducibilityHints : GetM Lean.ReducibilityHints := do | 2 => pure (.regular (← getTag0).size.toUInt32) | x => throw s!"invalid ReducibilityHints {x}" +/-- Order-independent merge for hint registration: alpha-equivalent + definitions share one constant address but may carry different + reducibility hints (e.g. one alias marked `@[reducible]`), and the + winner must not depend on registration order. Keeps the minimum + under `(tag, height)` with `opaque < abbrev < regular h` — + commutative, associative, idempotent. Mirrors Rust + `Env::register_hint`. -/ +def mergeHints (a b : Lean.ReducibilityHints) : Lean.ReducibilityHints := + let key : Lean.ReducibilityHints → Nat × Nat + | .opaque => (0, 0) + | .abbrev => (1, 0) + | .regular n => (2, n.toNat) + let (a₁, a₂) := key a + let (b₁, b₂) := key b + if b₁ < a₁ || (b₁ == a₁ && b₂ < a₂) then b else a + /-- Serialize DataValue with indexed addresses. OfString/OfNat/OfInt/OfSyntax use raw 32-byte addresses (blob addresses, not in name index). -/ def putDataValueIndexed (dv : DataValue) (idx : NameIndex) : PutM Unit := do @@ -1315,11 +1331,10 @@ def getExprMetaArenaIndexed (rev : NameReverseIndex) : GetM ExprMetaArena := do def putConstantMetaIndexed (cm : ConstantMeta) (idx : NameIndex) : PutM Unit := do match cm with | .empty => putU8 255 - | .defn name lvls hints all ctx arena typeRoot valueRoot => + | .defn name lvls all ctx arena typeRoot valueRoot => putU8 0 putIdx name idx putIdxVec lvls idx - putReducibilityHints hints putIdxVec all idx putIdxVec ctx idx putExprMetaArenaIndexed arena idx @@ -1389,13 +1404,12 @@ def getConstantMetaIndexed (rev : NameReverseIndex) : GetM ConstantMeta := do | 0 => let name ← getIdx rev let lvls ← getIdxVec rev - let hints ← getReducibilityHints let all ← getIdxVec rev let ctx ← getIdxVec rev let arena ← getExprMetaArenaIndexed rev let typeRoot := (← getTag0).size let valueRoot := (← getTag0).size - pure (.defn name lvls hints all ctx arena typeRoot valueRoot) + pure (.defn name lvls all ctx arena typeRoot valueRoot) | 1 => let name ← getIdx rev let lvls ← getIdxVec rev @@ -1530,9 +1544,10 @@ structure Env where as a strictly ascending leaf list; `Ix.Merkle.merkleRootCanonical` over it reproduces the root a `Claim.assumptions` commits to. -/ assumptions : Std.HashSet Address := {} - /-- Reducibility hints (§3, the canonical hint channel for anon/lazy - readers). When empty, `putEnv` derives the section from Named - `.defn` metadata at write time. Mirrors Rust `Env.anon_hints`. -/ + /-- Reducibility hints, keyed by constant address — the single home + for hints (they do not appear in `ConstantMeta`). The compiler + populates this map; `putEnv` serializes it as the hints section + and `getEnv` reads it back. Mirrors Rust `Env.anon_hints`. -/ anonHints : Std.HashMap Address Lean.ReducibilityHints := {} deriving Inhabited @@ -1871,28 +1886,10 @@ def putEnv (env : Env) : PutM Unit := do -- Section 3: anon_hints — the canonical hint channel for the -- anon/lazy readers, placed before the metadata sections so they can - -- stop right after it. When the in-memory map is empty (the compile - -- path: hints live in Named metadata), the section is derived from - -- `named`. `named` is sorted here (the §5 canonical order, by name - -- hash) and reused for §5 below; the sorted iteration makes the - -- first-wins dedup by constant address deterministic. Matches Rust - -- `Env::put`. - let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT - let hintPairs : Array (Address × Lean.ReducibilityHints) := - if env.anonHints.isEmpty then Id.run do - let mut seen : Std.HashSet Address := {} - let mut pairs : Array (Address × Lean.ReducibilityHints) := #[] - for (_, namedEntry) in named do - match namedEntry.constMeta with - | .defn _ _ hints _ _ _ _ _ => - if !seen.contains namedEntry.addr then - seen := seen.insert namedEntry.addr - pairs := pairs.push (namedEntry.addr, hints) - | _ => pure () - return pairs - else - env.anonHints.toList.toArray - let hintPairs := hintPairs.qsort fun a b => (compare a.1 b.1).isLT + -- stop right after it. Serialized straight from `env.anonHints`, the + -- single home for hints. Matches Rust `Env::put`. + let hintPairs := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT putTag0 ⟨hintPairs.size.toUInt64⟩ for (addr, hints) in hintPairs do Serialize.put addr @@ -1910,6 +1907,7 @@ def putEnv (env : Env) : PutM Unit := do putNameComponent name -- Section 5: Named (name Address -> Named with metadata) + let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT putTag0 ⟨named.size.toUInt64⟩ for (name, namedEntry) in named do -- Use the name's stored hash, which matches how it was stored in env.names @@ -2102,22 +2100,8 @@ def envSectionSizes (env : Env) : Nat × Nat × Nat × Nat × Nat × Nat := Id.r -- anon_hints section (mirrors putEnv's derive-from-named rule) let hintsBytes := runPut do - let named := env.named.toList.toArray.qsort fun a b => (compare a.1 b.1).isLT - let hintPairs : Array (Address × Lean.ReducibilityHints) := - if env.anonHints.isEmpty then Id.run do - let mut seen : Std.HashSet Address := {} - let mut pairs : Array (Address × Lean.ReducibilityHints) := #[] - for (_, namedEntry) in named do - match namedEntry.constMeta with - | .defn _ _ hints _ _ _ _ _ => - if !seen.contains namedEntry.addr then - seen := seen.insert namedEntry.addr - pairs := pairs.push (namedEntry.addr, hints) - | _ => pure () - return pairs - else - env.anonHints.toList.toArray - let hintPairs := hintPairs.qsort fun a b => (compare a.1 b.1).isLT + let hintPairs := env.anonHints.toList.toArray.qsort + fun a b => (compare a.1 b.1).isLT putTag0 ⟨hintPairs.size.toUInt64⟩ for (addr, hints) in hintPairs do Serialize.put addr @@ -2196,17 +2180,14 @@ structure RawConstSlice where len : UInt64 deriving Inhabited -/-- A named entry reduced to `name → addr` plus an encoded reducibility hint - (`hintKind`: 0 = none, 1 = opaque, 2 = abbrev, 3 = regular `hintVal`). -/ +/-- A named entry reduced to `name → addr`. -/ structure RawNamedLite where name : Ix.Name addr : Address - hintKind : UInt64 - hintVal : UInt64 deriving Inhabited /-- Metadata-light env returned by `rs_de_env_lazy`. Field order - matters: the Rust builder addresses constructor slots 0-4. -/ + matters: the Rust builder addresses constructor slots 0-5. -/ structure RawEnvLazy where consts : Array RawConstSlice named : Array RawNamedLite @@ -2215,33 +2196,25 @@ structure RawEnvLazy where main : Option Address := none /-- Bundle trust boundary (`Env.assumptions`), in header order. -/ assumptions : Array Address := #[] + /-- Reducibility hints (`Env.anonHints`) from the hints section. -/ + anonHints : Array (Address × Lean.ReducibilityHints) := #[] deriving Inhabited -/-- Decode an encoded reducibility hint into the `ConstantMeta` the check path - expects: a stripped `.defn` carrying just the hint (so `addEntries` finds it - via `.defn _ _ hints …`), or `.empty` for non-`Defn` entries. -/ -def RawNamedLite.toConstMeta (n : RawNamedLite) : ConstantMeta := - let mkDefn (h : Lean.ReducibilityHints) : ConstantMeta := - .defn default #[] h #[] #[] {} 0 0 - match n.hintKind with - | 1 => mkDefn .opaque - | 2 => mkDefn .abbrev - | 3 => mkDefn (.regular n.hintVal.toUInt32) - | _ => .empty - /-- Reconstruct an `Env` from a `RawEnvLazy` and the original buffer `buf`. Constants become `LazyConstant.ofSlice buf offset len` — zero-copy windows that share `buf`. `buf` MUST be the exact buffer passed to `rs_de_env_lazy` (offsets are relative to it). -/ def RawEnvLazy.toEnv (raw : RawEnvLazy) (buf : ByteArray) : Env := Id.run do let mut env : Env := { main := raw.main - assumptions := raw.assumptions.foldl (·.insert ·) {} } + assumptions := raw.assumptions.foldl (·.insert ·) {} + anonHints := raw.anonHints.foldl + (fun m (a, h) => m.insert a h) {} } for ⟨addr, offset, len⟩ in raw.consts do env := { env with consts := env.consts.insert addr (LazyConstant.ofSlice buf offset.toNat len.toNat) } for n in raw.named do - env := env.registerName n.name { addr := n.addr, constMeta := n.toConstMeta } + env := env.registerName n.name { addr := n.addr, constMeta := .empty } for ⟨addr, bytes⟩ in raw.blobs do env := { env with blobs := env.blobs.insert addr bytes } return env diff --git a/Tests/FFI/Ixon.lean b/Tests/FFI/Ixon.lean index b0107c69d..6f660965a 100644 --- a/Tests/FFI/Ixon.lean +++ b/Tests/FFI/Ixon.lean @@ -175,8 +175,8 @@ def constantMetaTests : TestSeq := let smallArena : ExprMetaArena := { nodes := #[.leaf, .app 0 0, .ref testAddr] } checkIO "ConstantMeta.empty" (roundtripIxonConstantMeta .empty == .empty) ++ checkIO "ConstantMeta.defn" (roundtripIxonConstantMeta - (.defn testAddr #[testAddr] .opaque #[] #[] smallArena 0 1) == - .defn testAddr #[testAddr] .opaque #[] #[] smallArena 0 1) ++ + (.defn testAddr #[testAddr] #[] #[] smallArena 0 1) == + .defn testAddr #[testAddr] #[] #[] smallArena 0 1) ++ checkIO "ConstantMeta.axio" (roundtripIxonConstantMeta (.axio testAddr #[] emptyArena 0) == .axio testAddr #[] emptyArena 0) ++ @@ -282,15 +282,16 @@ def envDiffTests : TestSeq := { info := .defn { kind := .defn, safety := .safe, lvls := 0, typ := .var 3, value } sharing := #[], refs := #[], univs := #[] } - -- Named metadata carries the hint; `anonHints` stays empty so the - -- writer derives §3 from the named `Def` metadata. + -- Hints live in `env.anonHints`, keyed by constant address; the + -- writer serializes that map as the hints section. let mkEnv (c : Constant) (h : Lean.ReducibilityHints) : Env := Id.run do let addr := Address.blake3 (serConstant c) let mut env : Env := {} env := env.storeConst addr c env := { env with names := RawEnv.addNameComponents env.names fooName } env := env.registerName fooName - { addr, constMeta := .defn fooName.getHash #[] h #[] #[] {} 0 0 } + { addr, constMeta := .defn fooName.getHash #[] #[] #[] {} 0 0 } + env := { env with anonHints := env.anonHints.insert addr h } return env let constA := mkConst (.var 0) let constB := mkConst (.var 1) @@ -356,16 +357,17 @@ def envDiffTests : TestSeq := d.namedChanged.size == 2 && (d.namedChanged.find? (·.name == "foo")).any (·.rippled == false) && (d.namedChanged.find? (·.name == "bar")).any (·.rippled == true)) ++ - -- Hints derive into anon §3, so a hint tweak is visible in the - -- default anon mode; the metadata carrying it only shows under meta. + -- Hints live only in the env-level hints section, so a hint tweak + -- is a pure `hintsChanged` row — no metadata difference in either + -- mode (the named entries are identical). test "EnvDiff: hint change visible in anon mode" ((runDiff envBase envHintChanged).any fun d => d.hintsChanged == #[(addrA, "regular(5)", "regular(6)")] && d.namedMetaOnly.isEmpty && d.namedChanged.isEmpty) ++ - test "EnvDiff: hint change flags metadata under meta mode" + test "EnvDiff: hint change is not a metadata change" ((runDiff envBase envHintChanged true).any fun d => - d.hintsChanged.size == 1 - && d.namedMetaOnly == #[("foo", #["meta.info"])]) ++ + d.hintsChanged.size == 1 && d.namedMetaOnly.isEmpty + && d.namedChanged.isEmpty) ++ test "EnvDiff: main change" ((runDiff { envBase with main := some addrA } envBase).any fun d => d.mainChanged == some (some addrA, none)) ++ diff --git a/Tests/Gen/Ixon.lean b/Tests/Gen/Ixon.lean index b001c0372..f89ebc961 100644 --- a/Tests/Gen/Ixon.lean +++ b/Tests/Gen/Ixon.lean @@ -356,7 +356,7 @@ def genConstantMeta : Gen ConstantMeta := do frequency [ (10, pure .empty), (15, ConstantMeta.defn <$> genAddress <*> genSmallArray genAddress - <*> genReducibilityHints <*> genSmallArray genAddress <*> genSmallArray genAddress + <*> genSmallArray genAddress <*> genSmallArray genAddress <*> pure arena <*> genRoot <*> genRoot), (15, ConstantMeta.axio <$> genAddress <*> genSmallArray genAddress <*> pure arena <*> genRoot), diff --git a/Tests/Ix/Compile.lean b/Tests/Ix/Compile.lean index 7efdf0003..f32d232d1 100644 --- a/Tests/Ix/Compile.lean +++ b/Tests/Ix/Compile.lean @@ -222,7 +222,7 @@ def testCrossImpl : TestSeq := IO.println s!" [{i}] {reprStr arena.nodes[i]!}" let dumpMeta (label : String) (cm : Ixon.ConstantMeta) : IO Unit := do match cm with - | .defn _ _ _ _ _ arena typeRoot valueRoot => do + | .defn _ _ _ _ arena typeRoot valueRoot => do dumpArena label "arena" arena IO.println s!" {label} typeRoot={typeRoot} valueRoot={valueRoot}" | .axio _ _ arena typeRoot => do @@ -261,10 +261,9 @@ def testCrossImpl : TestSeq := IO.println s!" variant: {leanTag}" -- Field-by-field comparison for common variants match leanCM, rustCM with - | .defn ln ll lh la lc larena ltr lvr, .defn rn rl rh ra rc rarena rtr rvr => do + | .defn ln ll la lc larena ltr lvr, .defn rn rl ra rc rarena rtr rvr => do if ln != rn then IO.println s!" name DIFFERS: Lean={ln} Rust={rn}" if ll != rl then IO.println s!" lvls DIFFERS: Lean={ll.size} Rust={rl.size}" - if lh != rh then IO.println s!" hints DIFFERS" if la != ra then IO.println s!" all DIFFERS: Lean={la} Rust={ra}" if lc != rc then IO.println s!" ctx DIFFERS: Lean={lc} Rust={rc}" if larena != rarena then IO.println s!" arena DIFFERS: Lean={larena.nodes.size} Rust={rarena.nodes.size}" diff --git a/crates/compile/src/compile.rs b/crates/compile/src/compile.rs index 63a4cc6b4..993314bde 100644 --- a/crates/compile/src/compile.rs +++ b/crates/compile/src/compile.rs @@ -20,8 +20,9 @@ use ix_common::env::{ AxiomVal, BinderInfo, ConstantInfo as LeanConstantInfo, ConstructorVal, DataValue as LeanDataValue, Env as LeanEnv, Expr as LeanExpr, ExprData, InductiveVal, Level, LevelData, Literal, Name, NameData, QuotVal, - RecursorRule as LeanRecursorRule, SourceInfo as LeanSourceInfo, - Substring as LeanSubstring, Syntax as LeanSyntax, SyntaxPreresolved, + RecursorRule as LeanRecursorRule, ReducibilityHints, + SourceInfo as LeanSourceInfo, Substring as LeanSubstring, + Syntax as LeanSyntax, SyntaxPreresolved, }; use ix_common::strong_ordering::SOrd; @@ -168,6 +169,13 @@ pub struct CompileState { /// right after `aux_gen::generate_aux_patches`. Blocks without nested /// auxiliaries simply aren't inserted. pub aux_perms: DashMap, + /// Reducibility hints per definition NAME, recorded by + /// `compile_definition` (the only place the Lean-side hints are in + /// scope — hints are not part of `ConstantMeta`). The constant + /// address a name resolves to isn't final until its `Named` entry is + /// registered, so [`Self::finalize_hints`] resolves this map through + /// `env.named` into `env.anon_hints` once compilation completes. + pub def_hints: DashMap, } /// Cached compiled expression with arena root index. @@ -228,6 +236,7 @@ impl Default for CompileState { brec_on_call_site_plans: Default::default(), below_call_site_plans: Default::default(), aux_perms: Default::default(), + def_hints: Default::default(), } } } @@ -264,6 +273,21 @@ impl CompileState { self.resolve_addr_aux(name, true) } + /// Resolve the per-name hints recorded by `compile_definition` into + /// `env.anon_hints`, keyed by each name's registered constant address + /// (`Named.addr` — the projection address for mutual-block members, + /// i.e. exactly the address the kernel looks hints up under). Runs + /// once after the scheduler drains: addresses aren't final until the + /// `Named` entries are registered. Alias collisions resolve through + /// `Env::register_hint`'s order-independent merge. + pub fn finalize_hints(&self) { + for entry in self.env.named.iter() { + if let Some(h) = self.def_hints.get(entry.key()) { + self.env.register_hint(entry.value().addr.clone(), *h.value()); + } + } + } + /// Promote a constant from `aux_name_to_addr` to `name_to_addr`, setting /// `Named.original` to the given `(orig_addr, orig_meta)` from the /// ephemeral no-aux compilation. The existing aux_gen `Named` entry keeps @@ -2219,7 +2243,6 @@ pub fn compile_definition( let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: name_addr, lvls: lvl_addrs, - hints: def.hints, all: all_addrs, ctx: ctx_addrs, arena, @@ -2227,6 +2250,7 @@ pub fn compile_definition( value_root, }); meta.meta_sharing = surgery_sharing; + stt.def_hints.insert(def.name.clone(), def.hints); Ok((data, meta)) } diff --git a/crates/compile/src/compile/env.rs b/crates/compile/src/compile/env.rs index 1afba1661..ca7b4d636 100644 --- a/crates/compile/src/compile/env.rs +++ b/crates/compile/src/compile/env.rs @@ -972,6 +972,8 @@ pub fn compile_env_with_options( }); } + stt.finalize_hints(); + if !*IX_QUIET { let total_elapsed = compile_start.elapsed().as_secs_f64(); eprintln!( diff --git a/crates/compile/src/decompile.rs b/crates/compile/src/decompile.rs index 536b13816..db94ac238 100644 --- a/crates/compile/src/decompile.rs +++ b/crates/compile/src/decompile.rs @@ -1351,14 +1351,25 @@ fn decompile_definition( dstt, )?; - let (hints, all) = match &meta.info { - ConstantMetaInfo::Def { hints, all, .. } => { + let all = match &meta.info { + ConstantMetaInfo::Def { all, .. } => { let all_names: Result, _> = all.iter().map(|a| decompile_name(a, stt)).collect(); - (*hints, all_names?) + all_names? }, - _ => (ReducibilityHints::Opaque, vec![]), + _ => vec![], }; + // Hints live in `env.anon_hints`, keyed by the constant address the + // name resolves to (for aux originals, the canonical address — the + // original was compiled from the same Lean definition, so the hints + // coincide). Absent entry → `Opaque`, matching the compiler's + // treatment of theorems and opaques. + let hints = stt + .env + .named + .get(&name) + .and_then(|n| stt.env.anon_hints.get(&n.value().addr).map(|r| *r)) + .unwrap_or(ReducibilityHints::Opaque); let cnst = ConstantVal { name, level_params, typ }; @@ -5473,7 +5484,6 @@ mod tests { let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: f_addr_name.clone(), lvls: vec![], - hints: ReducibilityHints::Opaque, all: vec![f_addr_name.clone()], ctx: vec![f_addr_name.clone()], arena, diff --git a/crates/ffi/src/lean.rs b/crates/ffi/src/lean.rs index dab1d3e74..20ae247ab 100644 --- a/crates/ffi/src/lean.rs +++ b/crates/ffi/src/lean.rs @@ -56,11 +56,11 @@ lean_ffi::lean_inductive! { LeanIxonRawEnv [ { num_obj: 8 } ]; // Lazy/anon deserialization (`rs_de_env_lazy`): zero-copy const windows - // (addr + offset + len), name->addr + hint, copied blobs, and the - // bundle header fields (main, assumptions). + // (addr + offset + len), name->addr, copied blobs, the bundle header + // fields (main, assumptions), and the hints-section pairs. LeanIxonRawConstSlice [ { num_obj: 1, num_64: 2 } ]; - LeanIxonRawNamedLite [ { num_obj: 2, num_64: 2 } ]; - LeanIxonRawEnvLazy [ { num_obj: 5 } ]; + LeanIxonRawNamedLite [ { num_obj: 2 } ]; + LeanIxonRawEnvLazy [ { num_obj: 6 } ]; // Env diff report (`rs_diff_envs`). Slot counts MUST match the Lean // structures in Ix/Ixon.lean (Ixon.EnvStats / NamedDiff / EnvDiff). @@ -110,7 +110,7 @@ lean_ffi::lean_inductive! { LeanIxonConstantMeta [ { }, // tag 0: empty (scalar) - { num_obj: 6, num_64: 2 }, // tag 1: defn + { num_obj: 5, num_64: 2 }, // tag 1: defn { num_obj: 3, num_64: 1 }, // tag 2: axio { num_obj: 3, num_64: 1 }, // tag 3: quot { num_obj: 6, num_64: 1 }, // tag 4: indc diff --git a/crates/ffi/src/lean_ixon/env.rs b/crates/ffi/src/lean_ixon/env.rs index e2a5dcd88..39ab800de 100644 --- a/crates/ffi/src/lean_ixon/env.rs +++ b/crates/ffi/src/lean_ixon/env.rs @@ -337,7 +337,7 @@ pub fn set_raw_env_bundle_fields( assumptions_arr.set(i, LeanIxAddress::build(addr)); } let mut hints: Vec<(Address, ReducibilityHints)> = - env.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect(); + env.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); let hints_arr = LeanArray::alloc(hints.len()); for (i, (addr, hint)) in hints.iter().enumerate() { @@ -486,7 +486,7 @@ pub fn ixon_env_to_decoded(env: &IxonEnv) -> Result { let mut assumptions: Vec
= env.assumptions.iter().cloned().collect(); assumptions.sort_unstable(); let mut anon_hints: Vec<(Address, ReducibilityHints)> = - env.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect(); + env.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); anon_hints.sort_unstable_by(|a, b| a.0.cmp(&b.0)); Ok(DecodedRawEnv { consts, @@ -608,17 +608,6 @@ pub extern "C" fn rs_de_env_anon( // closure it actually checks. See `Ix.Ixon.deEnvAnon`. // ============================================================================= -/// Encode an optional reducibility hint as `(kind, val)` for the FFI: -/// `0` = none (not a Defn), `1` = Opaque, `2` = Abbrev, `3` = Regular(val). -fn encode_hint(hint: &Option) -> (u64, u64) { - match hint { - None => (0, 0), - Some(ReducibilityHints::Opaque) => (1, 0), - Some(ReducibilityHints::Abbrev) => (2, 0), - Some(ReducibilityHints::Regular(n)) => (3, u64::from(*n)), - } -} - impl LeanIxonRawConstSlice { /// Build `Ixon.RawConstSlice { addr, offset, len }`. pub fn build(addr: &Address, offset: usize, len: usize) -> Self { @@ -631,19 +620,15 @@ impl LeanIxonRawConstSlice { } impl LeanIxonRawNamedLite { - /// Build `Ixon.RawNamedLite { name, addr, hintKind, hintVal }`. + /// Build `Ixon.RawNamedLite { name, addr }`. pub fn build( cache: &mut LeanBuildCache, name: &Name, addr: &Address, - hint: &Option, ) -> Self { - let (kind, val) = encode_hint(hint); let ctor = LeanIxonRawNamedLite::alloc(0); ctor.set_obj(0, LeanIxName::build(cache, name)); ctor.set_obj(1, LeanIxAddress::build(addr)); - ctor.set_num_64(0, kind); - ctor.set_num_64(1, val); ctor } } @@ -659,10 +644,7 @@ fn build_raw_env_lazy(index: &LazyIndex) -> LeanIxonRawEnvLazy { let named_arr = LeanArray::alloc(index.named.len()); for (i, n) in index.named.iter().enumerate() { - named_arr.set( - i, - LeanIxonRawNamedLite::build(&mut cache, &n.name, &n.addr, &n.hint), - ); + named_arr.set(i, LeanIxonRawNamedLite::build(&mut cache, &n.name, &n.addr)); } let blobs_arr = LeanArray::alloc(index.blobs.len()); @@ -679,12 +661,24 @@ fn build_raw_env_lazy(index: &LazyIndex) -> LeanIxonRawEnvLazy { assumptions_arr.set(i, LeanIxAddress::build(addr)); } + let hints_arr = LeanArray::alloc(index.hints.len()); + for (i, (addr, hint)) in index.hints.iter().enumerate() { + hints_arr.set( + i, + LeanProd::new( + LeanIxAddress::build(addr), + LeanIxReducibilityHints::build(hint), + ), + ); + } + let ctor = LeanIxonRawEnvLazy::alloc(0); ctor.set_obj(0, consts_arr); ctor.set_obj(1, named_arr); ctor.set_obj(2, blobs_arr); ctor.set_obj(3, main_obj); ctor.set_obj(4, assumptions_arr); + ctor.set_obj(5, hints_arr); ctor } diff --git a/crates/ffi/src/lean_ixon/meta.rs b/crates/ffi/src/lean_ixon/meta.rs index 5c8af7c00..844a68277 100644 --- a/crates/ffi/src/lean_ixon/meta.rs +++ b/crates/ffi/src/lean_ixon/meta.rs @@ -3,9 +3,8 @@ //! Includes: DataValue, KVMap, ExprMetaData, ExprMetaArena, ConstantMeta, Named, Comm use crate::lean::{ - LeanIxReducibilityHints, LeanIxonComm, LeanIxonConstantMeta, - LeanIxonDataValue, LeanIxonExprMetaArena, LeanIxonExprMetaData, - LeanIxonNamed, + LeanIxonComm, LeanIxonConstantMeta, LeanIxonDataValue, LeanIxonExprMetaArena, + LeanIxonExprMetaData, LeanIxonNamed, }; use ix_common::address::Address; use ix_common::env::BinderInfo; @@ -349,7 +348,7 @@ impl LeanIxonConstantMeta { /// | Variant | Tag | Obj fields | Scalar bytes | /// |---------|-----|-----------|-------------| /// | empty | 0 | 0 | 0 | - /// | defn | 1 | 6 (name, lvls, hints, all, ctx, arena) | 16 (2× u64) | + /// | defn | 1 | 5 (name, lvls, all, ctx, arena) | 16 (2× u64) | /// | axio | 2 | 3 (name, lvls, arena) | 8 (1× u64) | /// | quot | 3 | 3 (name, lvls, arena) | 8 (1× u64) | /// | indc | 4 | 6 (name, lvls, ctors, all, ctx, arena) | 8 (1× u64) | @@ -363,7 +362,6 @@ impl LeanIxonConstantMeta { ConstantMetaInfo::Def { name, lvls, - hints, all, ctx, arena, @@ -373,10 +371,9 @@ impl LeanIxonConstantMeta { let ctor = LeanIxonConstantMeta::alloc(1); ctor.set_obj(0, LeanIxAddress::build(name)); ctor.set_obj(1, LeanIxAddress::build_array(lvls)); - ctor.set_obj(2, LeanIxReducibilityHints::build(hints)); - ctor.set_obj(3, LeanIxAddress::build_array(all)); - ctor.set_obj(4, LeanIxAddress::build_array(ctx)); - ctor.set_obj(5, LeanIxonExprMetaArena::build(arena)); + ctor.set_obj(2, LeanIxAddress::build_array(all)); + ctor.set_obj(3, LeanIxAddress::build_array(ctx)); + ctor.set_obj(4, LeanIxonExprMetaArena::build(arena)); ctor.set_num_64(0, *type_root); ctor.set_num_64(1, *value_root); ctor @@ -486,18 +483,15 @@ impl LeanIxonConstantMeta { LeanIxAddress::from_borrowed(self.get_obj(0).as_byte_array()) .decode(); let lvls = decode_address_array(self.get_obj(1).as_array()); - let hints = - LeanIxReducibilityHints::new(self.get_obj(2).to_owned_ref()).decode(); - let all = decode_address_array(self.get_obj(3).as_array()); - let ctx = decode_address_array(self.get_obj(4).as_array()); + let all = decode_address_array(self.get_obj(2).as_array()); + let ctx = decode_address_array(self.get_obj(3).as_array()); let arena = - LeanIxonExprMetaArena::new(self.get_obj(5).to_owned_ref()).decode(); + LeanIxonExprMetaArena::new(self.get_obj(4).to_owned_ref()).decode(); let type_root = self.get_num_64(0); let value_root = self.get_num_64(1); ConstantMeta::new(ConstantMetaInfo::Def { name, lvls, - hints, all, ctx, arena, diff --git a/crates/ffi/src/lean_ixon/serialize.rs b/crates/ffi/src/lean_ixon/serialize.rs index e681802ac..ef29caafd 100644 --- a/crates/ffi/src/lean_ixon/serialize.rs +++ b/crates/ffi/src/lean_ixon/serialize.rs @@ -258,26 +258,18 @@ pub extern "C" fn rs_eq_env_serialization( return false; } - // Hints: the writers derive §3 from Named `Def` metadata when the - // explicit map is empty, so the parsed env's hints must equal the - // same derivation over the decoded RawEnv. + // Hints: `anon_hints` is the single home for hints (the writers + // serialize the map directly), so the parsed env's map must equal + // the decoded RawEnv's hints verbatim. let expected_hints: rustc_hash::FxHashMap< Address, ix_common::env::ReducibilityHints, - > = if decoded.anon_hints.is_empty() { - let mut derived = rustc_hash::FxHashMap::default(); - for rn in &decoded.named { - if let ixon::metadata::ConstantMetaInfo::Def { hints, .. } = - &rn.const_meta.info - { - derived.entry(rn.addr.clone()).or_insert(*hints); - } - } - derived - } else { - decoded.anon_hints.iter().cloned().collect() - }; - if rust_env.anon_hints != expected_hints { + > = decoded.anon_hints.iter().cloned().collect(); + let hints_match = rust_env.anon_hints.len() == expected_hints.len() + && expected_hints + .iter() + .all(|(a, h)| rust_env.anon_hints.get(a).map(|r| *r) == Some(*h)); + if !hints_match { if debug { eprintln!( "[rs_eq_env_serialization] anon_hints mismatch: rust={}, expected={}", diff --git a/crates/ixon/src/diff.rs b/crates/ixon/src/diff.rs index 70d2911c7..7981cc7af 100644 --- a/crates/ixon/src/diff.rs +++ b/crates/ixon/src/diff.rs @@ -1248,20 +1248,22 @@ fn diff_envs_impl( // Hints, joined on constants present in both envs. let shared = |addr: &Address| a.consts.contains_key(addr) && b.consts.contains_key(addr); - for (addr, ha) in &a.anon_hints { + for entry in a.anon_hints.iter() { + let (addr, ha) = (entry.key(), entry.value()); if !shared(addr) { continue; } - let hb = b.anon_hints.get(addr); - if hb != Some(ha) { + let hb = b.anon_hints.get(addr).map(|r| *r); + if hb != Some(*ha) { d.hints_changed.push(( addr.clone(), hint_label(Some(ha)), - hint_label(hb), + hint_label(hb.as_ref()), )); } } - for (addr, hb) in &b.anon_hints { + for entry in b.anon_hints.iter() { + let (addr, hb) = (entry.key(), entry.value()); if shared(addr) && !a.anon_hints.contains_key(addr) { d.hints_changed.push(( addr.clone(), @@ -1597,7 +1599,6 @@ mod tests { let info = ConstantMetaInfo::Def { name: Address::hash(b"nm"), lvls: vec![], - hints: ReducibilityHints::Regular(1), all: vec![], ctx: vec![], arena: ExprMeta::default(), @@ -2323,15 +2324,17 @@ mod tests { addr } - fn def_meta(name_addr: Address, hint: u32) -> ConstantMeta { + /// `variant` lands in `type_root`, so two calls with different + /// variants produce metas that differ while the constant (and its + /// address) stays identical — a metadata-only difference. + fn def_meta(name_addr: Address, variant: u32) -> ConstantMeta { ConstantMeta::new(ConstantMetaInfo::Def { name: name_addr, lvls: vec![], - hints: ReducibilityHints::Regular(hint), all: vec![], ctx: vec![], arena: ExprMeta::default(), - type_root: 0, + type_root: u64::from(variant), value_root: 0, }) } @@ -2347,8 +2350,8 @@ mod tests { #[test] fn meta_sweep_matches_full_reader() { let build = |foo_value: Arc, - foo_hint: u32, - monly_hint: u32, + foo_variant: u32, + monly_variant: u32, foo_original: bool| -> Vec { let env = Env::new(); @@ -2357,7 +2360,7 @@ mod tests { let c = defn_c(Expr::var(3), foo_value); let addr = store_canonical(&env, &c); let mut foo_named = - Named::new(addr.clone(), def_meta(foo_comp, foo_hint)); + Named::new(addr.clone(), def_meta(foo_comp, foo_variant)); if foo_original { foo_named.set_original(addr, ConstantMeta::default()); } @@ -2369,7 +2372,7 @@ mod tests { let stable_addr = store_canonical(&env, &stable); env.register_name( monly.clone(), - Named::new(stable_addr, def_meta(monly_comp, monly_hint)), + Named::new(stable_addr, def_meta(monly_comp, monly_variant)), ); let mut bytes = Vec::new(); env.put(&mut bytes).expect("put failed"); diff --git a/crates/ixon/src/env.rs b/crates/ixon/src/env.rs index c4a036a65..a4bb3a1b8 100644 --- a/crates/ixon/src/env.rs +++ b/crates/ixon/src/env.rs @@ -1,6 +1,6 @@ //! Environment for storing Ixon data. -use rustc_hash::{FxHashMap, FxHashSet}; +use rustc_hash::FxHashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -165,15 +165,14 @@ pub struct LazyConstSlice { pub len: usize, } -/// One named entry in a [`LazyIndex`]: just the `name → addr` mapping plus the -/// per-`Defn` reducibility hint (the only metadata the typecheck circuit -/// consumes). The heavy `ExprMetaArena` is parsed (to advance the cursor and -/// handle every metadata variant, e.g. `CallSite`) but discarded. +/// One named entry in a [`LazyIndex`]: just the `name → addr` mapping. +/// The heavy `ExprMetaArena` is parsed (to advance the cursor and +/// handle every metadata variant, e.g. `CallSite`) but discarded; +/// hints live on [`LazyIndex::hints`]. #[derive(Debug, Clone)] pub struct LazyNamed { pub name: Name, pub addr: Address, - pub hint: Option, } /// `IX_COMPILE_DEMOTE` (default **on**; set `0` to disable): store the @@ -227,7 +226,7 @@ pub struct LazyIndex { /// - `blobs`: Raw data (strings, nats, files) /// - `names`: Hash-consed Lean.Name components (Address -> Name) /// - `comms`: Cryptographic commitments (secrets) -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Env { /// Alpha-invariant constants: Address -> LazyConstant (raw bytes + /// optional materialized cache; see [`LazyConstant`]). @@ -240,20 +239,21 @@ pub struct Env { pub names: IxonMap, /// Cryptographic commitments: commitment Address -> Comm pub comms: IxonMap, - /// Reducibility hints sidecar harvested by [`Env::get_anon`] from the - /// otherwise-discarded Named section. Keyed by the constant's - /// projection/standalone address (i.e. `Named.addr` — the address the - /// kernel sees, **not** the name-hash address). Empty for envs loaded - /// via [`Env::get`] / [`Env::new`] / `store_*`; meta-mode ingress - /// pulls hints directly from `Named.meta` and ignores this field. + /// Reducibility hints, keyed by the constant's projection/standalone + /// address (i.e. `Named.addr` — the address the kernel sees, **not** + /// the name-hash address). The SINGLE home for hints: the compiler + /// registers them here ([`Env::register_hint`]), every writer + /// serializes this map as the file's hints section, and every reader + /// populates it from that section. Hints do not appear in `Named` + /// metadata. /// - /// Anon-mode ingress passes these hints through to - /// `ingress_defn` so the kernel's lazy-delta tiebreak - /// (`def_eq::def_rank_id`) sees realistic heights instead of the - /// constant `Regular(0)` fallback. Hints are performance advice — - /// supplying them in anon mode does not relax the kernel's - /// metadata-free correctness model. - pub anon_hints: FxHashMap, + /// Ingress passes these hints through to `ingress_defn` so the + /// kernel's lazy-delta tiebreak (`def_eq::def_rank_id`) sees + /// realistic heights instead of the constant `Regular(0)` fallback. + /// Hints are performance advice — supplying them does not relax the + /// kernel's metadata-free correctness model — and are intentionally + /// NOT covered by the consts merkle root. + pub anon_hints: IxonMap, /// Distinguished root constant for bundle envs; `None` for whole /// environments. A pointer, not a proof: nothing in the file /// authenticates it, so readers check `main ∈ consts` and consumers @@ -276,7 +276,7 @@ impl Env { blobs: IxonMap::new(), names: IxonMap::new(), comms: IxonMap::new(), - anon_hints: FxHashMap::default(), + anon_hints: IxonMap::new(), main: None, assumptions: FxHashSet::default(), } @@ -300,6 +300,38 @@ impl Env { self.blobs.get(addr).map(|r| r.clone()) } + /// Register a constant's reducibility hints (the compile-time + /// producer of [`Self::anon_hints`]; readers populate the map from + /// the file's hints section instead). + /// + /// Alpha-equivalent definitions share one constant address but may + /// carry different hints (e.g. one alias marked `@[reducible]`), and + /// parallel compile workers may race on the entry — so the winner + /// must not depend on arrival order. The merge keeps the minimum + /// under `(tag, height)` with `Opaque < Abbrev < Regular(h)`: + /// commutative, associative, idempotent, hence order-independent. + /// The Lean compiler mirror applies the same rule. + #[cfg(not(target_arch = "riscv64"))] + pub fn register_hint(&self, addr: Address, hints: ReducibilityHints) { + fn rank(h: &ReducibilityHints) -> (u8, u32) { + match h { + ReducibilityHints::Opaque => (0, 0), + ReducibilityHints::Abbrev => (1, 0), + ReducibilityHints::Regular(x) => (2, *x), + } + } + match self.anon_hints.entry(addr) { + dashmap::mapref::entry::Entry::Occupied(mut e) => { + if rank(&hints) < rank(e.get()) { + e.insert(hints); + } + }, + dashmap::mapref::entry::Entry::Vacant(e) => { + e.insert(hints); + }, + } + } + /// Store a structured constant under `addr`. /// /// Serializes the constant once. With demotion disabled, @@ -928,49 +960,6 @@ impl Env { } } -impl Clone for Env { - // `mut` is only needed on `riscv64` where `IxonMap` wraps `FxHashMap` and - // `insert` takes `&mut self`; on host `DashMap::insert` takes `&self`. - #[cfg_attr(not(target_arch = "riscv64"), allow(unused_mut))] - fn clone(&self) -> Self { - let mut consts = IxonMap::new(); - for entry in self.consts.iter() { - consts.insert(entry.key().clone(), entry.value().clone()); - } - - let mut named = IxonMap::new(); - for entry in self.named.iter() { - named.insert(entry.key().clone(), entry.value().clone()); - } - - let mut blobs = IxonMap::new(); - for entry in self.blobs.iter() { - blobs.insert(entry.key().clone(), entry.value().clone()); - } - - let mut names = IxonMap::new(); - for entry in self.names.iter() { - names.insert(entry.key().clone(), entry.value().clone()); - } - - let mut comms = IxonMap::new(); - for entry in self.comms.iter() { - comms.insert(entry.key().clone(), entry.value().clone()); - } - - Env { - consts, - named, - blobs, - names, - comms, - anon_hints: self.anon_hints.clone(), - main: self.main.clone(), - assumptions: self.assumptions.clone(), - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -1495,7 +1484,6 @@ mod tests { let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: name_addr.clone(), lvls: vec![], - hints: ReducibilityHints::Regular(3), all: vec![], ctx: vec![], arena: ExprMeta::default(), @@ -1504,6 +1492,7 @@ mod tests { }); meta.meta_refs.push(meta_blob.clone()); env.register_name(name.clone(), Named::new(a.clone(), meta)); + env.register_hint(a.clone(), ReducibilityHints::Regular(3)); let bundle = env.prune_to_closure(&a, &FxHashSet::default()).unwrap(); assert!(bundle.named.get(&name).is_some(), "named entry carried"); @@ -1522,12 +1511,15 @@ mod tests { bundle.get_blob(&Address::hash(b"Bundled")), Some(b"Bundled".to_vec()) ); - // Hints derive from the carried Def meta on serialization; the - // anon reader sees them without touching metadata sections. + // Hints are carried per constant by the prune; the anon reader + // sees them from the hints section without touching metadata. let mut buf = Vec::new(); bundle.put(&mut buf).unwrap(); let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); - assert_eq!(anon.anon_hints.get(&a), Some(&ReducibilityHints::Regular(3))); + assert_eq!( + anon.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(3)) + ); } /// Fixture for the streaming/anon prune tests: `a` (the main) named @@ -1552,7 +1544,6 @@ mod tests { let mut meta = ConstantMeta::new(ConstantMetaInfo::Def { name: foo_addr, lvls: vec![u_addr], - hints: ReducibilityHints::Regular(2), all: vec![], ctx: vec![], arena: ExprMeta::default(), @@ -1563,6 +1554,7 @@ mod tests { let mut foo_named = Named::new(a.clone(), meta); foo_named.set_original(b.clone(), ConstantMeta::default()); env.register_name(foo, foo_named); + env.register_hint(a.clone(), ReducibilityHints::Regular(2)); let bar = n("Bar"); let bar_addr = Address::from_blake3_hash(*bar.get_hash()); env.store_name(bar_addr, bar.clone()); @@ -1646,9 +1638,11 @@ mod tests { !bundle.consts.contains_key(&b), "metadata-only edges must not be walked in anon mode" ); - // §3 hints derive from the source's Named Def metadata at write - // time and ride the lazy env's anon_hints into the bundle. - assert_eq!(bundle.anon_hints.get(&a), Some(&ReducibilityHints::Regular(2))); + // Hints ride the lazy env's anon_hints into the bundle. + assert_eq!( + bundle.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(2)) + ); bundle.validate_closed().unwrap(); let mut buf = Vec::new(); @@ -1662,6 +1656,9 @@ mod tests { assert!(back.names.len() <= 1, "§4 carries at most the anon entry"); assert_eq!(back.main, Some(a.clone())); let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); - assert_eq!(anon.anon_hints.get(&a), Some(&ReducibilityHints::Regular(2))); + assert_eq!( + anon.anon_hints.get(&a).map(|r| *r), + Some(ReducibilityHints::Regular(2)) + ); } } diff --git a/crates/ixon/src/map.rs b/crates/ixon/src/map.rs index 748ebafb1..5d514782f 100644 --- a/crates/ixon/src/map.rs +++ b/crates/ixon/src/map.rs @@ -20,7 +20,7 @@ mod riscv_impl { use rustc_hash::FxHashMap; - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct IxonMap(FxHashMap); impl Default for IxonMap { diff --git a/crates/ixon/src/metadata.rs b/crates/ixon/src/metadata.rs index d6557f372..315be9750 100644 --- a/crates/ixon/src/metadata.rs +++ b/crates/ixon/src/metadata.rs @@ -121,7 +121,6 @@ pub enum ConstantMetaInfo { Def { name: Address, lvls: Vec
, - hints: ReducibilityHints, all: Vec
, ctx: Vec
, arena: ExprMeta, @@ -1154,20 +1153,10 @@ impl ConstantMetaInfo { ) -> Result<(), String> { match self { Self::Empty => put_u8(255, buf), - Self::Def { - name, - lvls, - hints, - all, - ctx, - arena, - type_root, - value_root, - } => { + Self::Def { name, lvls, all, ctx, arena, type_root, value_root } => { put_u8(0, buf); put_idx(name, idx, buf)?; put_idx_vec(lvls, idx, buf)?; - hints.put_ser(buf); put_idx_vec(all, idx, buf)?; put_idx_vec(ctx, idx, buf)?; arena.put_with(idx, buf)?; @@ -1260,7 +1249,6 @@ impl ConstantMetaInfo { 0 => Ok(Self::Def { name: get_idx(buf, rev)?, lvls: get_idx_vec(buf, rev)?, - hints: ReducibilityHints::get_ser(buf)?, all: get_idx_vec(buf, rev)?, ctx: get_idx_vec(buf, rev)?, arena: ExprMeta::get_with(buf, rev)?, @@ -1399,7 +1387,6 @@ mod tests { let meta = ConstantMeta::new(ConstantMetaInfo::Def { name: addr1.clone(), lvls: vec![addr2.clone(), addr3.clone()], - hints: ReducibilityHints::Regular(10), all: vec![addr1.clone()], ctx: vec![addr2.clone()], arena, diff --git a/crates/ixon/src/serialize.rs b/crates/ixon/src/serialize.rs index 436574f13..e4981eeaf 100644 --- a/crates/ixon/src/serialize.rs +++ b/crates/ixon/src/serialize.rs @@ -1424,47 +1424,17 @@ impl Env { // ───────────────────────────────────────────────────────────────────── // Section 3: anon_hints (Address -> ReducibilityHints) // - // The canonical hint channel for the anon/lazy readers, placed - // before the metadata sections so `get_anon`/`get_anon_mmap` can - // stop right after it (they never touch names/named/comms). When - // the in-memory map is empty (the compile path: hints live in - // Named metadata), the section is derived from `named` — the same - // harvest `get_anon` historically did at read time, moved to - // write time. Hints are performance-only advice and intentionally + // The canonical hint channel, placed before the metadata sections + // so `get_anon`/`get_anon_mmap` can stop right after it (they + // never touch names/named/comms). `anon_hints` is the single home + // for hints: the compiler registers them per constant address + // (`Env::register_hint`) and readers fill the map from this very + // section. Hints are performance-only advice and intentionally // NOT covered by the consts merkle root. - // - // `named_keys` is collected and sorted here (by name hash — the - // Section 5 canonical order) and reused for Section 5 below; the - // sorted iteration makes the first-wins dedup by constant address - // deterministic when alpha-equivalent names share one constant. // ───────────────────────────────────────────────────────────────────── let sec_start = std::time::Instant::now(); - let mut named_keys: Vec = - self.named.iter().map(|e| e.key().clone()).collect(); - #[cfg(not(target_arch = "riscv64"))] - named_keys.par_sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); - #[cfg(target_arch = "riscv64")] - named_keys.sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); let mut hint_pairs: Vec<(Address, ReducibilityHints)> = - if self.anon_hints.is_empty() { - let mut derived: FxHashMap = - FxHashMap::default(); - for name in &named_keys { - if let Some(entry) = self.named.get(name) - && let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &entry.value().meta().info - { - derived.entry(entry.value().addr.clone()).or_insert(*hints); - } - } - derived.into_iter().collect() - } else { - self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() - }; + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); put_u64(hint_pairs.len() as u64, buf); for (addr, hints) in &hint_pairs { @@ -1527,13 +1497,24 @@ impl Env { // with metadata arenas), so the streaming pattern's win is greatest // here: on Mathlib, avoiding the clone-into-Vec saves ~30 GB peak RAM. // - // `named_keys` was collected and sorted (by cached name hash bytes) - // up in Section 3, where the hint derivation needs the same - // canonical order. + // Key clone cost: a `Name` is `Arc`, so each clone is a + // single atomic refcount increment (<1s for 733k). let sec_start = std::time::Instant::now(); if !quiet { eprintln!("[Env::put] section 5/6 named: {} entries", self.named.len(),); } + let mut named_keys: Vec = + self.named.iter().map(|e| e.key().clone()).collect(); + // Sort by the cached name hash bytes — the section's canonical + // order (ascending name hash, which is exactly `Name`'s `Ord`). + #[cfg(not(target_arch = "riscv64"))] + named_keys.par_sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); + #[cfg(target_arch = "riscv64")] + named_keys.sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); let put_start = std::time::Instant::now(); put_u64(named_keys.len() as u64, buf); for name in &named_keys { @@ -1676,31 +1657,10 @@ impl Env { written += bytes.len() as u64; } } - // Section 3: anon_hints — the canonical hint channel. Always - // emitted; when the in-memory map is empty it is derived from Named - // `Def` metadata in hash-sorted name order, mirroring `Env::put` - // exactly (same iteration order → same `or_insert` winners). - let mut named_keys: Vec = - self.named.iter().map(|e| e.key().clone()).collect(); - named_keys.par_sort_unstable_by(|a, b| { - a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) - }); + // Section 3: anon_hints — the canonical hint channel, serialized + // straight from the map (see `Env::put`). let mut hint_pairs: Vec<(Address, ReducibilityHints)> = - if self.anon_hints.is_empty() { - let mut derived: FxHashMap = - FxHashMap::default(); - for name in &named_keys { - if let Some(entry) = self.named.get(name) - && let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &entry.value().meta().info - { - derived.entry(entry.value().addr.clone()).or_insert(*hints); - } - } - derived.into_iter().collect() - } else { - self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() - }; + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); put_u64(hint_pairs.len() as u64, &mut buf); for (addr, hints) in &hint_pairs { @@ -1726,6 +1686,11 @@ impl Env { // and re-encode through the name index). Chunks encode in parallel // into per-entry buffers and drain to the writer in order; the // chunk size bounds staged memory to a few MiB. + let mut named_keys: Vec = + self.named.iter().map(|e| e.key().clone()).collect(); + named_keys.par_sort_unstable_by(|a, b| { + a.get_hash().as_bytes().cmp(b.get_hash().as_bytes()) + }); put_u64(named_keys.len() as u64, &mut buf); emit!(); for chunk in named_keys.chunks(4096) { @@ -1957,13 +1922,9 @@ impl Env { index.consts.push(LazyConstSlice { addr, offset, len }); } - // Section 3: anon_hints — the canonical hint channel (the writer - // always emits it, deriving from Named metadata when needed), so - // `LazyNamed.hint` is a plain lookup instead of a metadata harvest. - // Kept verbatim on the index for full-reader parity. + // Section 3: anon_hints — kept verbatim on the index for + // full-reader parity. index.hints = read_hints_section(&mut buf)?; - let hints_map: FxHashMap = - index.hints.iter().cloned().collect(); // Section 4: Names — parsed to build the index for metadata // decoding. The reverse index is retained on the LazyIndex so §5 @@ -1983,10 +1944,10 @@ impl Env { } index.name_reverse_index = name_reverse_index; - // Section 5: Named — keep `name → addr` plus the §3 hint for that - // address; the full ConstantMeta (arena, CallSite, ...) is parsed - // then discarded. The section offset is recorded so a - // [`NamedMetaCursor`] can re-walk it without re-parsing §1–§4. + // Section 5: Named — keep `name → addr`; the full ConstantMeta + // (arena, CallSite, ...) is parsed then discarded. The section + // offset is recorded so a [`NamedMetaCursor`] can re-walk it + // without re-parsing the earlier sections. index.named_section_offset = data.len() - buf.len(); let num_named = get_u64(&mut buf)?; for _ in 0..num_named { @@ -1995,8 +1956,7 @@ impl Env { let name = names_lookup.get(&name_addr).cloned().ok_or_else(|| { format!("parse_lazy_index: missing name for addr {:?}", name_addr) })?; - let hint = hints_map.get(&named.addr).copied(); - index.named.push(LazyNamed { name, addr: named.addr, hint }); + index.named.push(LazyNamed { name, addr: named.addr }); } // Section 6: Comms — carried verbatim (tiny; empty for @@ -2404,23 +2364,10 @@ impl Env { } let consts_size = buf.len() - before_consts; - // Section 3: anon_hints (mirrors Env::put's derive-from-named rule) + // Section 3: anon_hints (serialized straight from the map) let before_hints = buf.len(); let mut hint_pairs: Vec<(Address, ReducibilityHints)> = - if self.anon_hints.is_empty() { - let mut derived: FxHashMap = - FxHashMap::default(); - for entry in self.named.iter() { - if let super::metadata::ConstantMetaInfo::Def { hints, .. } = - &entry.value().meta().info - { - derived.entry(entry.value().addr.clone()).or_insert(*hints); - } - } - derived.into_iter().collect() - } else { - self.anon_hints.iter().map(|(a, h)| (a.clone(), *h)).collect() - }; + self.anon_hints.iter().map(|e| (e.key().clone(), *e.value())).collect(); hint_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); put_u64(hint_pairs.len() as u64, &mut buf); for (addr, hints) in &hint_pairs { @@ -2730,11 +2677,8 @@ mod tests { env.assumptions.insert(Address::arbitrary(g)); } - // Explicit anon_hints (keyed by arbitrary addresses — the map is - // advisory). When left empty, `Env::put` derives §3 from Named - // metadata instead; gen_env metas are all `Empty`, so that - // derivation contributes nothing and roundtrip equality holds - // either way. + // anon_hints (keyed by arbitrary addresses — the map is advisory + // and the hints section is serialized straight from it). let num_hints = gen_range(g, 0..4); for _ in 0..num_hints { let variant: u8 = Arbitrary::arbitrary(g); @@ -2858,7 +2802,11 @@ mod tests { ); return false; } - if env.anon_hints != recovered.anon_hints { + let hints_match = env.anon_hints.len() == recovered.anon_hints.len() + && env.anon_hints.iter().all(|e| { + recovered.anon_hints.get(e.key()).map(|r| *r) == Some(*e.value()) + }); + if !hints_match { eprintln!( "anon_hints mismatch: {} vs {} entries", env.anon_hints.len(), @@ -3365,46 +3313,44 @@ mod tests { /// §3 is the canonical hint channel: with an empty `anon_hints` map /// the writer derives the section from Named `Def` metadata, and an /// env carrying the same hints explicitly serializes byte-identically. + /// The hints section round-trips through both the anon and full + /// readers, and `register_hint`'s merge is order-independent on + /// address collisions. #[test] - fn env_hints_derived_from_named_metadata() { - use crate::env::Named; - use crate::metadata::{ConstantMetaInfo, ExprMeta}; - + fn env_hints_roundtrip_and_merge_deterministic() { let env = Env::new(); let const_addr = store_canonical(&env, defn_const(vec![])); - let name = Name::str(Name::anon(), "hinted".to_string()); - let name_addr = Address::from_blake3_hash(*name.get_hash()); - env.names.insert(name_addr.clone(), name.clone()); - let meta = ConstantMeta::new(ConstantMetaInfo::Def { - name: name_addr, - lvls: vec![], - hints: ReducibilityHints::Regular(7), - all: vec![], - ctx: vec![], - arena: ExprMeta::default(), - type_root: 0, - value_root: 0, - }); - env.named.insert(name, Named::new(const_addr.clone(), meta)); + env.register_hint(const_addr.clone(), ReducibilityHints::Regular(7)); - // Empty map → §3 derived from Named at write time; the anon - // reader picks it up without ever parsing the Named section. let mut buf = Vec::new(); env.put(&mut buf).unwrap(); let anon = Env::get_anon(&mut buf.as_slice()).unwrap(); assert_eq!( - anon.anon_hints.get(&const_addr), - Some(&ReducibilityHints::Regular(7)) + anon.anon_hints.get(&const_addr).map(|r| *r), + Some(ReducibilityHints::Regular(7)) + ); + let full = Env::get(&mut buf.as_slice()).unwrap(); + assert_eq!( + full.anon_hints.get(&const_addr).map(|r| *r), + Some(ReducibilityHints::Regular(7)) ); - // Explicit map with identical content → identical bytes. - let mut env2 = env.clone(); - env2.anon_hints.insert(const_addr, ReducibilityHints::Regular(7)); - let mut buf2 = Vec::new(); - env2.put(&mut buf2).unwrap(); + // Alias collision: registration order must not affect the winner. + let a = Env::new(); + let ca = store_canonical(&a, defn_const(vec![])); + a.register_hint(ca.clone(), ReducibilityHints::Regular(9)); + a.register_hint(ca.clone(), ReducibilityHints::Abbrev); + let b = Env::new(); + let cb = store_canonical(&b, defn_const(vec![])); + b.register_hint(cb.clone(), ReducibilityHints::Abbrev); + b.register_hint(cb.clone(), ReducibilityHints::Regular(9)); + assert_eq!( + a.anon_hints.get(&ca).map(|r| *r), + b.anon_hints.get(&cb).map(|r| *r) + ); assert_eq!( - buf, buf2, - "derived and explicit hint sections should serialize identically" + a.anon_hints.get(&ca).map(|r| *r), + Some(ReducibilityHints::Abbrev) ); } } diff --git a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs index ead830954..cda5885ca 100644 --- a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs +++ b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs @@ -191,7 +191,7 @@ fn add_entries_parallel( // Hints come from env.anon_hints (sidecar). Collect per chunk. for addr in chunk { if let Some(h) = env.anon_hints.get(addr) { - p.hints.push((addr_key(addr), hint_to_g(h))); + p.hints.push((addr_key(addr), hint_to_g(&h))); } } p diff --git a/crates/kernel/src/anon_work.rs b/crates/kernel/src/anon_work.rs index 5ffe10377..138aacbc1 100644 --- a/crates/kernel/src/anon_work.rs +++ b/crates/kernel/src/anon_work.rs @@ -232,7 +232,7 @@ pub fn build_sub_env( #[cfg(not(target_arch = "riscv64"))] fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { let closure = closure_addrs(source, roots); - let mut sub = IxonEnv::new(); + let sub = IxonEnv::new(); for addr in &closure { if let Some(bytes) = source.get_const_bytes(addr) { sub.store_const_lazy(addr.clone(), bytes); @@ -241,10 +241,8 @@ fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { } // else: external ref absent from `source` — omit; stays an open assumption. // Carry the constant's reducibility hint so the guest reproduces vanilla - // kernel behavior. The sub-env has no Named section to derive the §3 - // hints from at serialization time, so populate the map explicitly; - // without hints the kernel forces `Regular(0)` and does extra def-eq - // work (the ~30% check overhead). + // kernel behavior; without hints the kernel forces `Regular(0)` and does + // extra def-eq work (the ~30% check overhead). if let Some(h) = source.anon_hints.get(addr) { sub.anon_hints.insert(addr.clone(), *h); } diff --git a/crates/kernel/src/ingress.rs b/crates/kernel/src/ingress.rs index ad6660daa..7d7ad8fe1 100644 --- a/crates/kernel/src/ingress.rs +++ b/crates/kernel/src/ingress.rs @@ -1396,45 +1396,30 @@ fn ingress_defn( // projection addresses; passing `Some(_)` skips the metadata-derived // `build_mut_ctx` call. Meta callers pass `None`. mut_ctx_override: Option>>, - // Anon callers may supply `Some(hints)` to override the default - // `Regular(0)` fall-through when the .ixe carries `Env::anon_hints` - // (harvested by `Env::get_anon` from the otherwise-discarded Named - // metadata). Meta callers pass `None` and pull hints from - // `meta.info` like usual. The override only takes effect when - // `meta.info` is not `Def` (i.e. anon path with empty meta). - hints_override: Option, ) -> Result, KConst)>, String> { let mut cache: ExprCache = FxHashMap::default(); let mut univ_cache: UnivCache = FxHashMap::default(); - let (level_params, arena, type_root, value_root, hints, safety, all_addrs) = - match &meta.info { - ConstantMetaInfo::Def { - lvls, - arena, - type_root, - value_root, - hints, - all, - .. - } => ( - resolve_level_params(lvls, names), - arena, - *type_root, - *value_root, - *hints, - def.safety, - all.clone(), - ), - _ => ( - vec![], - &DEFAULT_ARENA, - 0, - 0, - hints_override.unwrap_or(ReducibilityHints::Regular(0)), - def.safety, - vec![], - ), - }; + // Hints live only in `Env::anon_hints`, keyed by the constant's own + // address, in both anon and meta modes. Without an entry the kernel + // falls back to `Regular(0)` — always correct, just more def-eq work. + let hints = ixon_env + .anon_hints + .get(&self_id.addr) + .map_or(ReducibilityHints::Regular(0), |r| *r); + let safety = def.safety; + let (level_params, arena, type_root, value_root, all_addrs) = match &meta.info + { + ConstantMetaInfo::Def { + lvls, arena, type_root, value_root, all, .. + } => ( + resolve_level_params(lvls, names), + arena, + *type_root, + *value_root, + all.clone(), + ), + _ => (vec![], &DEFAULT_ARENA, 0, 0, vec![]), + }; let mut_ctx = match mut_ctx_override { Some(m) => m, @@ -1656,7 +1641,6 @@ fn ingress_standalone( intern, stats, None, - None, ), IxonCI::Axio(ax) => { @@ -2052,7 +2036,6 @@ fn ingress_muts_block( intern, stats, None, - None, )?); }, } @@ -3683,8 +3666,8 @@ fn drop_ixon_env(_ixon_env: IxonEnv, _quiet: bool) { #[cfg(not(target_arch = "riscv64"))] fn drop_ixon_env(ixon_env: IxonEnv, quiet: bool) { let total_start = Instant::now(); - // `anon_hints` is a small FxHashMap (one entry per Def from the .ixe's - // hints section); `main`/`assumptions` are a single address and a small + // `anon_hints` is small (one entry per Def from the .ixe's hints + // section); `main`/`assumptions` are a single address and a small // set. Dropping them inline alongside the bookkeeping below is // negligible compared to the DashMap dropdance. // `..` covers the env's private fields. @@ -4246,7 +4229,6 @@ fn ingress_anon_standalone( let empty_n2a: FxHashMap = FxHashMap::default(); let mut convert_stats = ConvertStats::new(false); let self_id: KId = KId::new(addr.clone(), ()); - let hints_override = anon_env.anon_hints.get(addr).copied(); let entries = match &constant.info { IxonCI::Defn(def) => ingress_defn::( @@ -4263,7 +4245,6 @@ fn ingress_anon_standalone( &mut kenv.intern, &mut convert_stats, Some(vec![self_id.clone()]), - hints_override, )?, IxonCI::Recr(rec) => ingress_recursor::( rec, @@ -4474,7 +4455,6 @@ pub fn ingress_anon_block( )?; let self_id = KId::::new(proj_addr.clone(), ()); member_kids.push(self_id.clone()); - let hints_override = anon_env.anon_hints.get(&proj_addr).copied(); let entries = ingress_defn::( def, @@ -4490,7 +4470,6 @@ pub fn ingress_anon_block( &mut kenv.intern, &mut convert_stats, Some(mut_ctx.clone()), - hints_override, )?; all_entries.extend(entries); }, diff --git a/docs/Ixon.md b/docs/Ixon.md index 52cedb067..558e4df4a 100644 --- a/docs/Ixon.md +++ b/docs/Ixon.md @@ -34,7 +34,7 @@ Every `Constant` in Ixon is serialized and hashed with blake3. The resulting 256 The Ixon format separates: - **Alpha-invariant data** (`Constant`): The mathematical content, hashed for addressing -- **Metadata** (`ConstantMeta`, `ExprMeta`): Names, binder info, reducibility hints—stored separately +- **Metadata** (`ConstantMeta`, `ExprMeta`): Names and binder info—stored separately (reducibility hints live at the environment level, in `Env::anon_hints`) This separation means cosmetic changes (renaming variables) don't change the constant's address. @@ -690,7 +690,7 @@ Per-constant metadata. Each variant stores a name, universe parameter names, an ```rust pub enum ConstantMeta { Empty, // tag 255 - Def { name, lvls, hints, all, ctx, + Def { name, lvls, all, ctx, arena, type_root, value_root }, // tag 0 Axio { name, lvls, arena, type_root }, // tag 1 Quot { name, lvls, arena, type_root }, // tag 2 @@ -706,7 +706,7 @@ pub enum ConstantMeta { | Tag | Variant | Payload | |-----|---------|---------| -| 0 | Def | name_idx, lvl_idxs, hints, all_idxs, ctx_idxs, arena, type_root, value_root | +| 0 | Def | name_idx, lvl_idxs, all_idxs, ctx_idxs, arena, type_root, value_root | | 1 | Axio | name_idx, lvl_idxs, arena, type_root | | 2 | Quot | name_idx, lvl_idxs, arena, type_root | | 3 | Indc | name_idx, lvl_idxs, ctor_idxs, all_idxs, ctx_idxs, arena, type_root | @@ -863,13 +863,13 @@ count (Tag0) [Address (32 bytes) + ReducibilityHints]* ``` -The canonical hint channel for the anon/lazy readers, keyed by -constant address and sorted ascending. Writers ALWAYS emit this -section: when the in-memory hint map is empty (the compile path — -hints live in Named metadata), the section is derived from the Named -entries' `Def` metadata at write time. Hints are performance-only -advice (the `Regular(0)` fallback is always correct) and are -intentionally outside the consts merkle root. +The canonical (and only) hint channel, keyed by constant address and +sorted ascending. Hints never appear in `ConstantMeta`: the compiler +registers each definition's hints into `Env::anon_hints` +(`Env::register_hint`, an order-independent merge on alias +collisions), writers serialize that map here, and readers load it +back. Hints are performance-only advice (the `Regular(0)` fallback is +always correct) and are intentionally outside the consts merkle root. **Section 4: Names** (Address → NameComponent, topologically sorted) ``` @@ -1312,7 +1312,6 @@ Named { meta: ConstantMeta::Def { name: addr_of_name("double"), lvls: [], - hints: ReducibilityHints::Regular(1), all: [addr_of_name("double")], ctx: [], arena: ExprMeta { nodes: [