Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions crates/tinymemory-api/src/provider/records.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -233,4 +233,34 @@ pub trait MemoryMaintenance: Send + Sync {
async fn latest_queue_failure(&self) -> Result<Option<QueueFailure>, MemoryError> {
Ok(None)
}

/// Whether a re-embedding backfill is still working through its rows.
///
/// **Driver-process-wide, and deliberately not store-scoped.** A driver
/// serving several stores in one process answers the same for all of them:
/// `true` means "a backfill is running somewhere in this driver", not "in
/// the store you asked about". That is why this is a member of its own
/// rather than a field on [`Self::queue_stats`] — a per-store snapshot is
/// asked of one bound provider, so a global sitting inside it reads as
/// per-store, and a caller has no way to find out otherwise. A global
/// behind a signature that says so is coarse; a global behind one that
/// does not is wrong.
///
/// Not derivable from the queue counts. A backfill runs as a chain that
/// re-enqueues itself, so between one link settling and the next being
/// written there is an instant with nothing ready, nothing running, and
/// the backfill nevertheless unfinished. The consumer is
/// absence-reasoning — deciding whether an empty semantic recall means
/// "nothing remembered" or "not embedded yet" — and it gets that wrong at
/// exactly that instant without this.
///
/// Defaulted to `false`: a driver that never backfills is not backfilling,
/// which is true of it rather than a refusal.
///
/// # Errors
///
/// Backend failures only.
async fn backfill_in_progress(&self) -> Result<bool, MemoryError> {
Ok(false)
}
}
2 changes: 1 addition & 1 deletion crates/tinymemory-bus/src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,7 @@
//! the members that carry them.
//!
//! TinyMemory ships as a loadable `TinyBus` module: `crates/tinymemory-module`
//! exports one object with 93 members on it, built as a `cdylib`. A host that
//! exports one object with 94 members on it, built as a `cdylib`. A host that
//! loads it — OpenHuman — can call into it but cannot `use` anything out of it,
//! so the payload vocabulary has to be published as an ordinary library. This
//! is that library.
Expand Down
6 changes: 5 additions & 1 deletion crates/tinymemory-bus/src/names.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -158,6 +158,9 @@ pub mod methods {
pub const QUEUE_STATS: &str = "QueueStats";
/// `LatestQueueFailure` — the most recent terminal queue failure.
pub const LATEST_QUEUE_FAILURE: &str = "LatestQueueFailure";
/// `BackfillInProgress` — whether a re-embedding backfill is still running
/// anywhere in the driver's process.
pub const BACKFILL_IN_PROGRESS: &str = "BackfillInProgress";

// The people store: ranking, handles, scores and interactions.
/// `ListPeople` — list people.
Expand DownExpand Up@@ -247,7 +250,7 @@ pub mod methods {
/// The order matters: `tinybus`'s `Interface::members()` returns declaration
/// order, and the module compares the two sequences directly rather than as
/// sets, so a reordering is caught alongside an addition or a removal.
pub const METHODS: [&str; 93] = [
pub const METHODS: [&str; 94] = [
methods::DRIVER_ID,
methods::CAPABILITIES,
methods::HEALTH,
Expand DownExpand Up@@ -303,6 +306,7 @@ pub const METHODS: [&str; 93] = [
methods::STORE_STATS,
methods::QUEUE_STATS,
methods::LATEST_QUEUE_FAILURE,
methods::BACKFILL_IN_PROGRESS,
methods::LIST_PEOPLE,
methods::GET_PERSON,
methods::RESOLVE_HANDLE,
Expand Down
1 change: 1 addition & 0 deletions crates/tinymemory-module/src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -311,6 +311,7 @@ mod exports {
"StoreStats",
"QueueStats",
"LatestQueueFailure",
"BackfillInProgress",
],
signals = [],
// The host's embedder is deliberately NOT declared as `requires`. That
Expand Down
7 changes: 7 additions & 0 deletions crates/tinymemory-module/src/service/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -915,6 +915,13 @@ impl MemoryService {
.map_err(|error| into_bus_error(&error))
}

async fn backfill_in_progress(&self) -> BusResult<bool> {
require_family!(self, as_maintenance, Capability::Maintenance)
.backfill_in_progress()
.await
.map_err(|error| into_bus_error(&error))
}

// ── People ──────────────────────────────────────────────────────────────

/// Known people, ranked by closeness.
Expand Down
2 changes: 1 addition & 1 deletion crates/tinymemory-module/src/service/test.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,7 @@ fn the_served_members_are_exactly_the_published_contract() {
.map(|member| (*member).to_string())
.collect();

// Reported as differences rather than as a 93-element inequality, so the
// Reported as differences rather than as a 94-element inequality, so the
// failure names the method that moved instead of printing both lists.
let missing: Vec<&String> = served.iter().filter(|m| !published.contains(m)).collect();
assert!(
Expand Down
1 change: 1 addition & 0 deletions crates/tinymemory-module/tests/module_e2e.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -675,6 +675,7 @@ const EXPECTED_METHODS: &[&str] = &[
"StoreStats",
"QueueStats",
"LatestQueueFailure",
"BackfillInProgress",
];

#[tokio::test]
Expand Down
13 changes: 13 additions & 0 deletions crates/tinymemory-tinycortex/src/engine/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1348,6 +1348,19 @@ impl MemoryMaintenance for TinycortexProvider {
.await
}

async fn backfill_in_progress(&self) -> Result<bool, MemoryError> {
// A process-global the backfill chain owns, not a column — and not one
// this engine can narrow, since `tinymemory_core::queue` tracks the
// chain for the process rather than per workspace. No `blocking`: it is
// an atomic load, not a query.
//
// The contract member says process-wide in its own signature, which is
// the whole reason this is not a `QueueStats` field: `queue_stats` is
// asked of one bound provider for one store, and a global answered
// there would read as store-scoped to every caller.
Ok(tinymemory_core::queue::backfill_in_progress())
}

async fn compact(&self) -> Result<MaintenanceReport, MemoryError> {
let (examined, changed) =
blocking(self.config.clone(), "compact memory queue", move |config| {
Expand Down
54 changes: 54 additions & 0 deletions crates/tinymemory-tinycortex/tests/full_provider_conformance.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -335,6 +335,60 @@ async fn retrying_moves_parked_work_back_to_ready_and_counts_it() {
);
}

/// The backfill flag is answered through the contract, and its scope is the
/// driver's process rather than the store.
///
/// Not derivable from `queue_stats`: a backfill chain has an instant between
/// links with nothing ready and nothing running and the work unfinished, which
/// is exactly when a caller reasoning about an empty recall needs it. What
/// this pins is that the member reports the engine's state rather than the
/// trait's `false` default — the failure that closes a re-embed modal while
/// the driver is still preparing work.
#[tokio::test]
async fn the_backfill_flag_is_reported_through_the_contract() {
use tinymemory_api::provider::MemoryMaintenance;

// The flag is a process-global. A test that sets it puts it back on every
// path, including a failing assertion, or it leaks into whatever runs next
// in this process.
struct Restore;
impl Drop for Restore {
fn drop(&mut self) {
tinymemory_core::queue::set_backfill_in_progress(false);
}
}

let workspace = tempfile::tempdir().expect("workspace");
let provider = provider_over(workspace.path());

assert!(
!provider
.backfill_in_progress()
.await
.expect("read the flag"),
"a store with no backfill running says so"
);

let _restore = Restore;
tinymemory_core::queue::set_backfill_in_progress(true);
assert!(
provider
.backfill_in_progress()
.await
.expect("read the flag"),
"the member reports the engine's state, not the trait's default"
);

// The pairing is the point: at this instant the counts alone would tell a
// caller the queue is finished.
let stats = provider.queue_stats(None).await.expect("queue stats");
assert_eq!(
(stats.ready, stats.running),
(0, 0),
"precondition: nothing ready and nothing running, yet a backfill is up"
);
}

/// A reported failure carries the success watermark that decides whether it is
/// still worth showing.
///
Expand Down
Loading