From 856df64599c16f49ab5f32e9430cb335dd8f6f7d Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 24 Aug 2026 11:52:55 +0530 Subject: [PATCH] Answer the backfill flag through the contract, at the scope it actually has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `backfill_in_progress` was briefly a `QueueStats` field in #86 and was dropped there for a good reason: it is a process-global, while `open_store` serves one store per memory subtree in a single process, so a per-store snapshot carrying it promised a scope it does not have. Leaving the host to call the global directly answers that objection and creates the other half of the same problem. Once re-embedding runs in the module — which is what `Maintenance::reembed` now does — the host-linked static is not merely coarse, it is always `false`, because a `cdylib` has its own statics. A frontend poll deciding whether to keep a re-embed modal open then closes it during exactly the gap the flag exists to cover. Both objections are satisfied by the same shape, which is neither of the two tried so far: a standalone `Maintenance` member, documented as driver-process- wide and explicitly not store-scoped. The honest scope lives in the signature instead of being implied away by the struct it sits in, and the answer comes from the bound driver, which is where the backfill actually runs. It stays undivable from the counts: a backfill re-enqueues itself, so between one link settling and the next being written there is an instant with nothing ready, nothing running and the work unfinished. The conformance test pins that instant — flag up, both counts zero — and restores the global on drop so a failing assertion cannot leak into the rest of the process. Registered in all four lists: the interface block, the module manifest, `tinymemory_bus::METHODS` with its length, and the loader's `EXPECTED_METHODS`. `every_declared_method_is_actually_routed` passes against the built cdylib. Co-Authored-By: Claude Opus 5 --- crates/tinymemory-api/src/provider/records.rs | 30 +++++++++++ crates/tinymemory-bus/src/lib.rs | 2 +- crates/tinymemory-bus/src/names.rs | 6 ++- crates/tinymemory-module/src/lib.rs | 1 + crates/tinymemory-module/src/service/mod.rs | 7 +++ crates/tinymemory-module/src/service/test.rs | 2 +- crates/tinymemory-module/tests/module_e2e.rs | 1 + .../tinymemory-tinycortex/src/engine/mod.rs | 13 +++++ .../tests/full_provider_conformance.rs | 54 +++++++++++++++++++ 9 files changed, 113 insertions(+), 3 deletions(-) diff --git a/crates/tinymemory-api/src/provider/records.rs b/crates/tinymemory-api/src/provider/records.rs index 80dc064..0e3038f 100644 --- a/crates/tinymemory-api/src/provider/records.rs +++ b/crates/tinymemory-api/src/provider/records.rs @@ -233,4 +233,34 @@ pub trait MemoryMaintenance: Send + Sync { async fn latest_queue_failure(&self) -> Result, MemoryError> { Ok(None) } + + /// Whether a re-embedding backfill is still working through its rows. + /// + /// **Driver-process-wide, and deliberately not store-scoped.** A driver + /// serving several stores in one process answers the same for all of them: + /// `true` means "a backfill is running somewhere in this driver", not "in + /// the store you asked about". That is why this is a member of its own + /// rather than a field on [`Self::queue_stats`] — a per-store snapshot is + /// asked of one bound provider, so a global sitting inside it reads as + /// per-store, and a caller has no way to find out otherwise. A global + /// behind a signature that says so is coarse; a global behind one that + /// does not is wrong. + /// + /// Not derivable from the queue counts. A backfill runs as a chain that + /// re-enqueues itself, so between one link settling and the next being + /// written there is an instant with nothing ready, nothing running, and + /// the backfill nevertheless unfinished. The consumer is + /// absence-reasoning — deciding whether an empty semantic recall means + /// "nothing remembered" or "not embedded yet" — and it gets that wrong at + /// exactly that instant without this. + /// + /// Defaulted to `false`: a driver that never backfills is not backfilling, + /// which is true of it rather than a refusal. + /// + /// # Errors + /// + /// Backend failures only. + async fn backfill_in_progress(&self) -> Result { + Ok(false) + } } diff --git a/crates/tinymemory-bus/src/lib.rs b/crates/tinymemory-bus/src/lib.rs index 8c1ec93..cc00d88 100644 --- a/crates/tinymemory-bus/src/lib.rs +++ b/crates/tinymemory-bus/src/lib.rs @@ -2,7 +2,7 @@ //! the members that carry them. //! //! TinyMemory ships as a loadable `TinyBus` module: `crates/tinymemory-module` -//! exports one object with 93 members on it, built as a `cdylib`. A host that +//! exports one object with 94 members on it, built as a `cdylib`. A host that //! loads it — OpenHuman — can call into it but cannot `use` anything out of it, //! so the payload vocabulary has to be published as an ordinary library. This //! is that library. diff --git a/crates/tinymemory-bus/src/names.rs b/crates/tinymemory-bus/src/names.rs index ea13c0a..a9f7161 100644 --- a/crates/tinymemory-bus/src/names.rs +++ b/crates/tinymemory-bus/src/names.rs @@ -158,6 +158,9 @@ pub mod methods { pub const QUEUE_STATS: &str = "QueueStats"; /// `LatestQueueFailure` — the most recent terminal queue failure. pub const LATEST_QUEUE_FAILURE: &str = "LatestQueueFailure"; + /// `BackfillInProgress` — whether a re-embedding backfill is still running + /// anywhere in the driver's process. + pub const BACKFILL_IN_PROGRESS: &str = "BackfillInProgress"; // The people store: ranking, handles, scores and interactions. /// `ListPeople` — list people. @@ -247,7 +250,7 @@ pub mod methods { /// The order matters: `tinybus`'s `Interface::members()` returns declaration /// order, and the module compares the two sequences directly rather than as /// sets, so a reordering is caught alongside an addition or a removal. -pub const METHODS: [&str; 93] = [ +pub const METHODS: [&str; 94] = [ methods::DRIVER_ID, methods::CAPABILITIES, methods::HEALTH, @@ -303,6 +306,7 @@ pub const METHODS: [&str; 93] = [ methods::STORE_STATS, methods::QUEUE_STATS, methods::LATEST_QUEUE_FAILURE, + methods::BACKFILL_IN_PROGRESS, methods::LIST_PEOPLE, methods::GET_PERSON, methods::RESOLVE_HANDLE, diff --git a/crates/tinymemory-module/src/lib.rs b/crates/tinymemory-module/src/lib.rs index 9e5647e..5812ff2 100644 --- a/crates/tinymemory-module/src/lib.rs +++ b/crates/tinymemory-module/src/lib.rs @@ -311,6 +311,7 @@ mod exports { "StoreStats", "QueueStats", "LatestQueueFailure", + "BackfillInProgress", ], signals = [], // The host's embedder is deliberately NOT declared as `requires`. That diff --git a/crates/tinymemory-module/src/service/mod.rs b/crates/tinymemory-module/src/service/mod.rs index 420bc2d..e0c75f8 100644 --- a/crates/tinymemory-module/src/service/mod.rs +++ b/crates/tinymemory-module/src/service/mod.rs @@ -915,6 +915,13 @@ impl MemoryService { .map_err(|error| into_bus_error(&error)) } + async fn backfill_in_progress(&self) -> BusResult { + require_family!(self, as_maintenance, Capability::Maintenance) + .backfill_in_progress() + .await + .map_err(|error| into_bus_error(&error)) + } + // ── People ────────────────────────────────────────────────────────────── /// Known people, ranked by closeness. diff --git a/crates/tinymemory-module/src/service/test.rs b/crates/tinymemory-module/src/service/test.rs index 0fc1818..45bc41d 100644 --- a/crates/tinymemory-module/src/service/test.rs +++ b/crates/tinymemory-module/src/service/test.rs @@ -544,7 +544,7 @@ fn the_served_members_are_exactly_the_published_contract() { .map(|member| (*member).to_string()) .collect(); - // Reported as differences rather than as a 93-element inequality, so the + // Reported as differences rather than as a 94-element inequality, so the // failure names the method that moved instead of printing both lists. let missing: Vec<&String> = served.iter().filter(|m| !published.contains(m)).collect(); assert!( diff --git a/crates/tinymemory-module/tests/module_e2e.rs b/crates/tinymemory-module/tests/module_e2e.rs index fc32dba..9e751c5 100644 --- a/crates/tinymemory-module/tests/module_e2e.rs +++ b/crates/tinymemory-module/tests/module_e2e.rs @@ -675,6 +675,7 @@ const EXPECTED_METHODS: &[&str] = &[ "StoreStats", "QueueStats", "LatestQueueFailure", + "BackfillInProgress", ]; #[tokio::test] diff --git a/crates/tinymemory-tinycortex/src/engine/mod.rs b/crates/tinymemory-tinycortex/src/engine/mod.rs index 5af3f48..b460709 100644 --- a/crates/tinymemory-tinycortex/src/engine/mod.rs +++ b/crates/tinymemory-tinycortex/src/engine/mod.rs @@ -1348,6 +1348,19 @@ impl MemoryMaintenance for TinycortexProvider { .await } + async fn backfill_in_progress(&self) -> Result { + // A process-global the backfill chain owns, not a column — and not one + // this engine can narrow, since `tinymemory_core::queue` tracks the + // chain for the process rather than per workspace. No `blocking`: it is + // an atomic load, not a query. + // + // The contract member says process-wide in its own signature, which is + // the whole reason this is not a `QueueStats` field: `queue_stats` is + // asked of one bound provider for one store, and a global answered + // there would read as store-scoped to every caller. + Ok(tinymemory_core::queue::backfill_in_progress()) + } + async fn compact(&self) -> Result { let (examined, changed) = blocking(self.config.clone(), "compact memory queue", move |config| { diff --git a/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs b/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs index d3cf0f9..67fbb88 100644 --- a/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs +++ b/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs @@ -335,6 +335,60 @@ async fn retrying_moves_parked_work_back_to_ready_and_counts_it() { ); } +/// The backfill flag is answered through the contract, and its scope is the +/// driver's process rather than the store. +/// +/// Not derivable from `queue_stats`: a backfill chain has an instant between +/// links with nothing ready and nothing running and the work unfinished, which +/// is exactly when a caller reasoning about an empty recall needs it. What +/// this pins is that the member reports the engine's state rather than the +/// trait's `false` default — the failure that closes a re-embed modal while +/// the driver is still preparing work. +#[tokio::test] +async fn the_backfill_flag_is_reported_through_the_contract() { + use tinymemory_api::provider::MemoryMaintenance; + + // The flag is a process-global. A test that sets it puts it back on every + // path, including a failing assertion, or it leaks into whatever runs next + // in this process. + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + tinymemory_core::queue::set_backfill_in_progress(false); + } + } + + let workspace = tempfile::tempdir().expect("workspace"); + let provider = provider_over(workspace.path()); + + assert!( + !provider + .backfill_in_progress() + .await + .expect("read the flag"), + "a store with no backfill running says so" + ); + + let _restore = Restore; + tinymemory_core::queue::set_backfill_in_progress(true); + assert!( + provider + .backfill_in_progress() + .await + .expect("read the flag"), + "the member reports the engine's state, not the trait's default" + ); + + // The pairing is the point: at this instant the counts alone would tell a + // caller the queue is finished. + let stats = provider.queue_stats(None).await.expect("queue stats"); + assert_eq!( + (stats.ready, stats.running), + (0, 0), + "precondition: nothing ready and nothing running, yet a backfill is up" + ); +} + /// A reported failure carries the success watermark that decides whether it is /// still worth showing. ///