From 3186c2bb7efcdb4f604f917800bc5281c7f84a5f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 16:28:42 -0500 Subject: [PATCH 01/39] refactor: remove old pattern --- migrations/postgres/0001_initial.sql | 9 - migrations/sqlite/0001_initial.sql | 9 - src/commit_builder/mod.rs | 40 ----- src/hashmap_repo/repository.rs | 50 +----- src/lib.rs | 6 +- src/outbox/table.rs | 3 +- src/postgres_repo/mod.rs | 111 +----------- src/read_model/in_memory.rs | 98 +++-------- src/read_model/mod.rs | 13 +- src/read_model/session.rs | 93 +--------- src/repository/async_repository.rs | 6 - src/sqlite_repo/mod.rs | 111 +----------- tests/async_repository/main.rs | 50 +----- tests/distributed_read_model/main.rs | 33 +--- .../projection_service/handlers/checkout.rs | 4 +- .../projection_service/handlers/seat.rs | 4 +- .../projection_service/mod.rs | 4 +- .../projection_service/service.rs | 4 - tests/distributed_read_model_board/main.rs | 15 +- .../projections_service/handlers/board.rs | 2 - .../projections_service/mod.rs | 2 - tests/hashmap_repository_conformance/main.rs | 7 +- .../read_models.rs | 30 +--- tests/postgres_repository/main.rs | 55 +----- .../main.rs | 166 ------------------ .../read_model_relationship_includes/main.rs | 4 - tests/read_model_session/main.rs | 10 +- tests/sqlite_repository/main.rs | 49 +----- 28 files changed, 93 insertions(+), 895 deletions(-) delete mode 100644 tests/read_model_distributed_idempotency/main.rs diff --git a/migrations/postgres/0001_initial.sql b/migrations/postgres/0001_initial.sql index f314f5415..5f72ec627 100644 --- a/migrations/postgres/0001_initial.sql +++ b/migrations/postgres/0001_initial.sql @@ -24,15 +24,6 @@ CREATE INDEX IF NOT EXISTS aggregate_events_event_version_idx CREATE INDEX IF NOT EXISTS aggregate_events_recorded_at_idx ON aggregate_events (recorded_at); -CREATE TABLE IF NOT EXISTS read_model_processed_messages ( - consumer_name text NOT NULL, - message_id text NOT NULL, - processed_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (consumer_name, message_id), - CHECK (consumer_name <> ''), - CHECK (message_id <> '') -); - CREATE TABLE IF NOT EXISTS aggregate_snapshots ( aggregate_type text NOT NULL, aggregate_id text NOT NULL, diff --git a/migrations/sqlite/0001_initial.sql b/migrations/sqlite/0001_initial.sql index b69bcf2c3..d24e68aa6 100644 --- a/migrations/sqlite/0001_initial.sql +++ b/migrations/sqlite/0001_initial.sql @@ -24,15 +24,6 @@ CREATE INDEX IF NOT EXISTS aggregate_events_event_version_idx CREATE INDEX IF NOT EXISTS aggregate_events_recorded_at_idx ON aggregate_events (recorded_at); -CREATE TABLE IF NOT EXISTS read_model_processed_messages ( - consumer_name TEXT NOT NULL, - message_id TEXT NOT NULL, - processed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (consumer_name, message_id), - CHECK (consumer_name <> ''), - CHECK (message_id <> '') -); - CREATE TABLE IF NOT EXISTS aggregate_snapshots ( aggregate_type TEXT NOT NULL, aggregate_id TEXT NOT NULL, diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index 25b9c464c..be6a20c11 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -1005,25 +1005,6 @@ mod tests { ); } - #[test] - fn invalid_session_plan_does_not_commit_aggregate() { - let repo = RecordingBatchRepo::default(); - let mut session = crate::read_model::ReadModelWritePlanBuilder::new(); - session.mark_processed("", "message-1"); - let mut agg = TestAggregate::default(); - agg.touch().unwrap(); - - let err = SyncReadModelWritePlanCommitExt::read_models_sync(&repo, session) - .commit_sync(&mut agg) - .unwrap_err(); - - assert!( - matches!(err, RepositoryError::Model(message) if message.contains("processed-message")) - ); - assert_eq!(agg.entity().committed_version(), 0); - assert!(repo.entity_ids.borrow().is_empty()); - } - #[test] fn commit_builder_failure_does_not_mark_aggregate_committed() { let repo = RecordingBatchRepo { @@ -1160,25 +1141,4 @@ mod tests { ] ); } - - #[tokio::test] - async fn async_invalid_read_model_plan_does_not_commit_aggregate() { - let repo = RecordingAsyncBatchRepo::default(); - let mut read_models = crate::read_model::ReadModelWritePlanBuilder::new(); - read_models.mark_processed("", "message-1"); - let mut agg = TestAggregate::default(); - agg.touch().unwrap(); - - let err = repo - .read_models(read_models) - .commit(&mut agg) - .await - .unwrap_err(); - - assert!( - matches!(err, RepositoryError::Model(message) if message.contains("processed-message")) - ); - assert_eq!(agg.entity().committed_version(), 0); - assert!(repo.stream_ids.lock().unwrap().is_empty()); - } } diff --git a/src/hashmap_repo/repository.rs b/src/hashmap_repo/repository.rs index 1fce794d3..283598861 100644 --- a/src/hashmap_repo/repository.rs +++ b/src/hashmap_repo/repository.rs @@ -188,11 +188,6 @@ impl AsyncTransactionalCommit for HashMapRepository { .relational_rows .write() .map_err(|_| RepositoryError::LockPoisoned("async read model write"))?; - let mut processed_messages = self - .model_store - .processed_messages - .write() - .map_err(|_| RepositoryError::LockPoisoned("async processed-message write"))?; let mut snapshot_storage = self .snapshot_store .storage @@ -205,7 +200,6 @@ impl AsyncTransactionalCommit for HashMapRepository { let mut staged_events = storage.clone(); let mut staged_rows = relational_rows.clone(); - let mut staged_processed_messages = processed_messages.clone(); let mut staged_snapshots = snapshot_storage.clone(); let mut staged_outbox = outbox_storage.clone(); @@ -229,17 +223,7 @@ impl AsyncTransactionalCommit for HashMapRepository { } for plan in batch.read_model_plans { - let outcome = apply_read_model_write_plan( - plan, - &mut staged_rows, - &mut staged_processed_messages, - )?; - if let Some(mark) = outcome.duplicate_message() { - return Err(RepositoryError::Model(format!( - "processed message already handled by consumer `{}`: `{}`", - mark.consumer_name, mark.message_id - ))); - } + apply_read_model_write_plan(plan, &mut staged_rows)?; } for write in batch.snapshots { @@ -260,7 +244,6 @@ impl AsyncTransactionalCommit for HashMapRepository { *storage = staged_events; *relational_rows = staged_rows; - *processed_messages = staged_processed_messages; *snapshot_storage = staged_snapshots; *outbox_storage = staged_outbox; @@ -287,11 +270,6 @@ impl TransactionalCommit for HashMapRepository { .relational_rows .write() .map_err(|_| RepositoryError::LockPoisoned("read model write"))?; - let mut processed_messages = self - .model_store - .processed_messages - .write() - .map_err(|_| RepositoryError::LockPoisoned("processed-message write"))?; let mut snapshot_storage = self .snapshot_store .storage @@ -304,7 +282,6 @@ impl TransactionalCommit for HashMapRepository { let mut staged_events = storage.clone(); let mut staged_rows = relational_rows.clone(); - let mut staged_processed_messages = processed_messages.clone(); let mut staged_snapshots = snapshot_storage.clone(); let mut staged_outbox = outbox_storage.clone(); @@ -330,17 +307,7 @@ impl TransactionalCommit for HashMapRepository { } for plan in batch.read_model_plans { - let outcome = apply_read_model_write_plan( - plan, - &mut staged_rows, - &mut staged_processed_messages, - )?; - if let Some(mark) = outcome.duplicate_message() { - return Err(RepositoryError::Model(format!( - "processed message already handled by consumer `{}`: `{}`", - mark.consumer_name, mark.message_id - ))); - } + apply_read_model_write_plan(plan, &mut staged_rows)?; } for write in batch.snapshots { @@ -363,7 +330,6 @@ impl TransactionalCommit for HashMapRepository { // Phase 3: Publish staged state only after all validation and staging succeeds. *storage = staged_events; *relational_rows = staged_rows; - *processed_messages = staged_processed_messages; *snapshot_storage = staged_snapshots; *outbox_storage = staged_outbox; @@ -498,10 +464,6 @@ impl ReadModelWritePlanStore for HashMapRepository { ) -> Result { ReadModelWritePlanStore::commit_write_plan(&self.model_store, plan) } - - fn is_processed(&self, consumer_name: &str, message_id: &str) -> Result { - ReadModelWritePlanStore::is_processed(&self.model_store, consumer_name, message_id) - } } impl AsyncReadModelWritePlanStore for HashMapRepository { @@ -515,14 +477,6 @@ impl AsyncReadModelWritePlanStore for HashMapRepository { ) -> impl Future> + Send + '_ { async move { ReadModelWritePlanStore::commit_write_plan(self, plan) } } - - fn is_processed_async<'a>( - &'a self, - consumer_name: &'a str, - message_id: &'a str, - ) -> impl Future> + Send + 'a { - async move { ReadModelWritePlanStore::is_processed(self, consumer_name, message_id) } - } } impl RelationalReadModelQueryStore for HashMapRepository { diff --git a/src/lib.rs b/src/lib.rs index 169ec1b76..fca4b85a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,9 +114,9 @@ pub use queued_repo::{ // Read models: projections and read-optimized views pub use read_model::{ ColumnDef, ColumnType, DeleteRowMutation, ExpectedVersion, ForeignKey, InMemoryReadModelStore, - IndexDef, PatchMode, PatchRowMutation, PrimaryKey, ProcessedMessageMark, ReadModel, - ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, - ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMigrationArtifact, ReadModelMutation, + IndexDef, PatchMode, PatchRowMutation, PrimaryKey, ReadModel, ReadModelAdapterCapabilities, + ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, + ReadModelLoadRequest, ReadModelMigrationArtifact, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, ReadModelSchemaAdapter, ReadModelSchemaAdapterCapabilities, ReadModelSchemaBootstrap, ReadModelSchemaIssue, ReadModelSchemaIssueKind, ReadModelSchemaRegistry, ReadModelSchemaVerification, diff --git a/src/outbox/table.rs b/src/outbox/table.rs index ade893856..9492e52d3 100644 --- a/src/outbox/table.rs +++ b/src/outbox/table.rs @@ -70,7 +70,7 @@ pub fn outbox_message_insert_plan( expected_version: ExpectedVersion::NotExists, mode: RowWriteMode::Insert, }); - Ok(TableWritePlan::new(vec![mutation], Vec::new())) + Ok(TableWritePlan::new(vec![mutation])) } pub(crate) fn validate_outbox_message_table_write( @@ -211,7 +211,6 @@ mod tests { let plan = outbox_message_insert_plan(&message).unwrap(); - assert!(plan.processed_messages.is_empty()); assert_eq!(plan.mutations.len(), 1); let TableMutation::UpsertRow(mutation) = &plan.mutations[0] else { panic!("outbox insert should lower to a table row mutation"); diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 9e9edea1e..3ae670e83 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -8,7 +8,7 @@ reason = "async trait impls return impl Future + Send to preserve public Send bounds" )] -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use std::future::Future; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -25,11 +25,11 @@ use crate::outbox_worker::{ }; use crate::read_model::{ column_name_for, key_fingerprint, validate_key, validate_row_values, ColumnDef, ColumnType, - DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ProcessedMessageMark, - ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, - ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, - ReadModelSchema, ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, - RowValue, RowValues, RowWriteMode, Versioned, + DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, + ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, + ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, + ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, RowValue, + RowValues, RowWriteMode, Versioned, }; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, @@ -312,13 +312,7 @@ impl AsyncTransactionalCommit for PostgresRepository { } for plan in batch.read_model_plans { - let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - if let Some(mark) = outcome.duplicate_message() { - return Err(RepositoryError::Model(format!( - "processed message already handled by consumer `{}`: `{}`", - mark.consumer_name, mark.message_id - ))); - } + apply_read_model_write_plan_in_tx(&mut tx, plan).await?; } for write in batch.snapshots { @@ -355,20 +349,10 @@ impl AsyncReadModelWritePlanStore for PostgresRepository { validate_sql_write_plan(&plan)?; let mut tx = begin_read_model_tx(&self.pool).await?; let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - if outcome.was_applied() { - commit_read_model_tx(tx).await?; - } + commit_read_model_tx(tx).await?; Ok(outcome) } } - - fn is_processed_async<'a>( - &'a self, - consumer_name: &'a str, - message_id: &'a str, - ) -> impl Future> + Send + 'a { - async move { processed_message_exists_pool(&self.pool, consumer_name, message_id).await } - } } impl AsyncRelationalReadModelQueryStore for PostgresRepository { @@ -876,7 +860,6 @@ fn sql_read_model_capabilities() -> ReadModelAdapterCapabilities { relational_rows: true, sparse_patches: true, deletes: true, - processed_messages: true, } } @@ -902,14 +885,6 @@ async fn apply_read_model_write_plan_in_tx( ) -> Result { validate_sql_write_plan(&plan)?; - let mut marks_in_plan = HashSet::with_capacity(plan.processed_messages.len()); - for mark in &plan.processed_messages { - let key = processed_message_key(mark); - if !marks_in_plan.insert(key) || processed_message_exists_in_tx(tx, mark).await? { - return Ok(ReadModelCommitOutcome::skipped_duplicate(mark.clone())); - } - } - for mutation in plan.mutations { match mutation { ReadModelMutation::UpsertRow(mutation) => { @@ -924,16 +899,6 @@ async fn apply_read_model_write_plan_in_tx( } } - for mark in plan.processed_messages { - let result = insert_processed_message_in_tx(tx, &mark).await; - if let Err(err) = result { - if is_postgres_unique_violation(&err) { - return Ok(ReadModelCommitOutcome::skipped_duplicate(mark)); - } - return Err(read_model_storage_error("insert processed message", err)); - } - } - Ok(ReadModelCommitOutcome::applied()) } @@ -1542,66 +1507,6 @@ fn belongs_to_target_column( Ok(target_schema.primary_key.columns[0].clone()) } -async fn processed_message_exists_pool( - pool: &PgPool, - consumer_name: &str, - message_id: &str, -) -> Result { - let row = sqlx::query( - r#" - SELECT 1 - FROM read_model_processed_messages - WHERE consumer_name = $1 AND message_id = $2 - "#, - ) - .bind(consumer_name) - .bind(message_id) - .fetch_optional(pool) - .await - .map_err(|err| read_model_storage_error("load processed message", err))?; - Ok(row.is_some()) -} - -async fn processed_message_exists_in_tx( - tx: &mut Transaction<'_, Postgres>, - mark: &ProcessedMessageMark, -) -> Result { - let row = sqlx::query( - r#" - SELECT 1 - FROM read_model_processed_messages - WHERE consumer_name = $1 AND message_id = $2 - "#, - ) - .bind(&mark.consumer_name) - .bind(&mark.message_id) - .fetch_optional(&mut **tx) - .await - .map_err(|err| read_model_storage_error("load processed message", err))?; - Ok(row.is_some()) -} - -async fn insert_processed_message_in_tx( - tx: &mut Transaction<'_, Postgres>, - mark: &ProcessedMessageMark, -) -> Result<(), sqlx::Error> { - sqlx::query( - r#" - INSERT INTO read_model_processed_messages (consumer_name, message_id) - VALUES ($1, $2) - "#, - ) - .bind(&mark.consumer_name) - .bind(&mark.message_id) - .execute(&mut **tx) - .await?; - Ok(()) -} - -fn processed_message_key(mark: &ProcessedMessageMark) -> (String, String) { - (mark.consumer_name.clone(), mark.message_id.clone()) -} - fn initial_row_version() -> u64 { 1 } diff --git a/src/read_model/in_memory.rs b/src/read_model/in_memory.rs index 8114f599a..53c9447a4 100644 --- a/src/read_model/in_memory.rs +++ b/src/read_model/in_memory.rs @@ -4,16 +4,16 @@ reason = "async trait impls return impl Future + Send to preserve public Send bounds" )] -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::sync::{Arc, RwLock}; use super::session::{column_name_for, key_fingerprint, validate_key, validate_row_values}; use super::{ - ExpectedVersion, PatchMode, ProcessedMessageMark, ReadModelAdapterCapabilities, - ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, - ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, - ReadModelSchemaRegistry, ReadModelWritePlan, ReadModelWritePlanStore, RelationalReadModel, + ExpectedVersion, PatchMode, ReadModelAdapterCapabilities, ReadModelCommitOutcome, + ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, + ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, ReadModelSchemaRegistry, + ReadModelWritePlan, ReadModelWritePlanStore, RelationalReadModel, RelationalReadModelQueryStore, RelationshipDef, RelationshipKind, RowKey, RowValue, RowValues, RowWriteMode, Versioned, }; @@ -25,8 +25,6 @@ pub(crate) struct StoredRow { pub(crate) version: u64, } -pub(crate) type ProcessedMessageSet = HashSet<(String, String)>; - pub(crate) const INITIAL_MODEL_VERSION: u64 = 1; /// Return the next optimistic version for a read model row. @@ -52,18 +50,9 @@ fn relational_capabilities() -> ReadModelAdapterCapabilities { pub(crate) fn apply_read_model_write_plan( plan: ReadModelWritePlan, staged_rows: &mut HashMap, - staged_processed_messages: &mut ProcessedMessageSet, ) -> Result { plan.validate_for(&relational_capabilities())?; - let mut marks_in_plan = HashSet::with_capacity(plan.processed_messages.len()); - for mark in &plan.processed_messages { - let key = processed_message_key(mark); - if staged_processed_messages.contains(&key) || !marks_in_plan.insert(key) { - return Ok(ReadModelCommitOutcome::skipped_duplicate(mark.clone())); - } - } - for mutation in plan.mutations { match mutation { ReadModelMutation::UpsertRow(mutation) => { @@ -149,17 +138,9 @@ pub(crate) fn apply_read_model_write_plan( } } - for mark in plan.processed_messages { - staged_processed_messages.insert(processed_message_key(&mark)); - } - Ok(ReadModelCommitOutcome::applied()) } -fn processed_message_key(mark: &ProcessedMessageMark) -> (String, String) { - (mark.consumer_name.clone(), mark.message_id.clone()) -} - fn relational_storage_key(table_name: &str, key: &RowKey) -> String { format!("{}:{}", table_name, key_fingerprint(key)) } @@ -252,7 +233,6 @@ fn concurrency_conflict( #[derive(Clone)] pub struct InMemoryReadModelStore { pub(crate) relational_rows: Arc>>, - pub(crate) processed_messages: Arc>, schema_registry: Arc>, } @@ -267,7 +247,6 @@ impl InMemoryReadModelStore { pub fn new() -> Self { Self { relational_rows: Arc::new(RwLock::new(HashMap::new())), - processed_messages: Arc::new(RwLock::new(HashSet::new())), schema_registry: Arc::new(RwLock::new(ReadModelSchemaRegistry::new())), } } @@ -312,31 +291,16 @@ impl ReadModelWritePlanStore for InMemoryReadModelStore { .relational_rows .write() .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; - let mut processed_messages = self - .processed_messages - .write() - .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; let mut staged_rows = relational_rows.clone(); - let mut staged_processed_messages = processed_messages.clone(); - let outcome = - apply_read_model_write_plan(plan, &mut staged_rows, &mut staged_processed_messages)?; + let outcome = apply_read_model_write_plan(plan, &mut staged_rows)?; if outcome.was_applied() { *relational_rows = staged_rows; - *processed_messages = staged_processed_messages; } Ok(outcome) } - - fn is_processed(&self, consumer_name: &str, message_id: &str) -> Result { - let processed_messages = self - .processed_messages - .read() - .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; - Ok(processed_messages.contains(&(consumer_name.to_string(), message_id.to_string()))) - } } impl AsyncReadModelWritePlanStore for InMemoryReadModelStore { @@ -350,14 +314,6 @@ impl AsyncReadModelWritePlanStore for InMemoryReadModelStore { ) -> impl Future> + Send + '_ { async move { ReadModelWritePlanStore::commit_write_plan(self, plan) } } - - fn is_processed_async<'a>( - &'a self, - consumer_name: &'a str, - message_id: &'a str, - ) -> impl Future> + Send + 'a { - async move { ReadModelWritePlanStore::is_processed(self, consumer_name, message_id) } - } } #[derive(Clone)] @@ -651,7 +607,7 @@ mod tests { } #[test] - fn relational_write_plan_upserts_rows_and_marks_processed() { + fn relational_write_plan_upserts_rows() { let store = InMemoryReadModelStore::new(); let schema = test_row_schema(); let key = RowKey::new([("id", RowValue::String("row-1".into()))]); @@ -659,19 +615,15 @@ mod tests { values.insert("id", RowValue::String("row-1".into())); let outcome = store - .commit_write_plan(ReadModelWritePlan::new( - vec![ReadModelMutation::UpsertRow(RowMutation { + .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( + RowMutation { schema: schema.clone(), key: key.clone(), values, expected_version: ExpectedVersion::Any, mode: RowWriteMode::Upsert, - })], - vec![ProcessedMessageMark { - consumer_name: "projection".into(), - message_id: "event-1".into(), - }], - )) + }, + )])) .unwrap(); let row = store .relational_rows @@ -687,7 +639,6 @@ mod tests { row.values.get("id"), Some(&RowValue::String("row-1".into())) ); - assert!(store.is_processed("projection", "event-1").unwrap()); } #[test] @@ -699,28 +650,26 @@ mod tests { values.insert("id", RowValue::String("row-1".into())); store - .commit_write_plan(ReadModelWritePlan::new( - vec![ReadModelMutation::UpsertRow(RowMutation { + .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( + RowMutation { schema: schema.clone(), key: key.clone(), values, expected_version: ExpectedVersion::Any, mode: RowWriteMode::Upsert, - })], - Vec::new(), - )) + }, + )])) .unwrap(); store - .commit_write_plan(ReadModelWritePlan::new( - vec![ReadModelMutation::PatchRow(PatchRowMutation { + .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::PatchRow( + PatchRowMutation { schema: schema.clone(), key: key.clone(), patch: RowPatch::new().set("id", RowValue::String("row-1".into())), expected_version: ExpectedVersion::Exact(1), mode: PatchMode::UpdateExisting, - })], - Vec::new(), - )) + }, + )])) .unwrap(); let version = store .relational_rows @@ -732,14 +681,13 @@ mod tests { assert_eq!(version, 2); store - .commit_write_plan(ReadModelWritePlan::new( - vec![ReadModelMutation::DeleteRow(DeleteRowMutation { + .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::DeleteRow( + DeleteRowMutation { schema: schema.clone(), key: key.clone(), expected_version: ExpectedVersion::Exact(2), - })], - Vec::new(), - )) + }, + )])) .unwrap(); assert!(!store .relational_rows diff --git a/src/read_model/mod.rs b/src/read_model/mod.rs index 9e377557a..c0d3351c2 100644 --- a/src/read_model/mod.rs +++ b/src/read_model/mod.rs @@ -16,11 +16,12 @@ //! from `commit`. //! //! Distributed projectors can commit a write plan directly against a read-model -//! adapter and mark messages processed in the same adapter transaction: +//! adapter. Projection handlers should make those writes idempotent so bus +//! retries can safely replay the same message: //! //! ```ignore //! let mut read_models = ReadModelWritePlanBuilder::new(); -//! read_models.upsert(&view)?.mark_processed("projection", event_id); +//! read_models.upsert(&view)?; //! let outcome = read_models.commit(&read_store)?; //! ``` @@ -116,9 +117,9 @@ pub use schema::{ #[cfg(any(feature = "postgres", feature = "sqlite"))] pub(crate) use session::{column_name_for, key_fingerprint, validate_key, validate_row_values}; pub use session::{ - DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ProcessedMessageMark, - ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelIncludeRows, ReadModelLoadGraph, - ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelWorkspace, - ReadModelWorkspaceExt, ReadModelWritePlan, ReadModelWritePlanBuilder, ReadModelWritePlanStore, + DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, + ReadModelCommitOutcome, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, + ReadModelMutation, ReadModelQueryCapabilities, ReadModelWorkspace, ReadModelWorkspaceExt, + ReadModelWritePlan, ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModelQueryStore, RowMutation, RowPatch, RowWriteMode, }; diff --git a/src/read_model/session.rs b/src/read_model/session.rs index 2dc0b8f04..a7829760c 100644 --- a/src/read_model/session.rs +++ b/src/read_model/session.rs @@ -43,7 +43,6 @@ pub struct ReadModelAdapterCapabilities { pub relational_rows: bool, pub sparse_patches: bool, pub deletes: bool, - pub processed_messages: bool, } impl Default for ReadModelAdapterCapabilities { @@ -52,43 +51,21 @@ impl Default for ReadModelAdapterCapabilities { relational_rows: true, sparse_patches: true, deletes: true, - processed_messages: true, } } } /// Result of applying a standalone read-model write plan. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ReadModelCommitOutcome { - applied: bool, - duplicate_message: Option, -} +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReadModelCommitOutcome; impl ReadModelCommitOutcome { pub fn applied() -> Self { - Self { - applied: true, - duplicate_message: None, - } - } - - pub fn skipped_duplicate(mark: ProcessedMessageMark) -> Self { - Self { - applied: false, - duplicate_message: Some(mark), - } + Self } pub fn was_applied(&self) -> bool { - self.applied - } - - pub fn was_skipped(&self) -> bool { - !self.applied - } - - pub fn duplicate_message(&self) -> Option<&ProcessedMessageMark> { - self.duplicate_message.as_ref() + true } } @@ -100,8 +77,6 @@ pub trait ReadModelWritePlanStore: Send + Sync { &self, plan: ReadModelWritePlan, ) -> Result; - - fn is_processed(&self, consumer_name: &str, message_id: &str) -> Result; } /// A request an adapter can satisfy with a primary-key read plus explicit includes. @@ -341,33 +316,19 @@ impl ReadModelMutation { } } -/// A processed-message marker staged with read-model writes. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProcessedMessageMark { - pub consumer_name: String, - pub message_id: String, -} - /// Deterministic unit-of-work output for relational read-model adapters. #[derive(Clone, Debug, Default, PartialEq)] pub struct ReadModelWritePlan { pub mutations: Vec, - pub processed_messages: Vec, } impl ReadModelWritePlan { - pub fn new( - mutations: Vec, - processed_messages: Vec, - ) -> Self { - Self { - mutations, - processed_messages, - } + pub fn new(mutations: Vec) -> Self { + Self { mutations } } pub fn is_empty(&self) -> bool { - self.mutations.is_empty() && self.processed_messages.is_empty() + self.mutations.is_empty() } pub fn validate(&self) -> Result<(), ReadModelError> { @@ -407,20 +368,6 @@ impl ReadModelWritePlan { } } - if !capabilities.processed_messages && !self.processed_messages.is_empty() { - return Err(ReadModelError::Metadata( - "read-model adapter does not support processed-message marks".into(), - )); - } - - for mark in &self.processed_messages { - if mark.consumer_name.is_empty() || mark.message_id.is_empty() { - return Err(ReadModelError::Metadata( - "processed-message marks require consumer name and message id".into(), - )); - } - } - Ok(()) } } @@ -441,7 +388,6 @@ struct StagedMutation { #[derive(Clone, Debug, Default)] pub struct ReadModelWritePlanBuilder { mutations: Vec, - processed_messages: Vec, expected_versions: BTreeMap, next_sequence: u64, } @@ -452,7 +398,7 @@ impl ReadModelWritePlanBuilder { } pub fn is_empty(&self) -> bool { - self.mutations.is_empty() && self.processed_messages.is_empty() + self.mutations.is_empty() } pub fn load(&self, key: RowKey) -> Result @@ -608,18 +554,6 @@ impl ReadModelWritePlanBuilder { self.delete::(model.primary_key()?) } - pub fn mark_processed( - &mut self, - consumer_name: impl Into, - message_id: impl Into, - ) -> &mut Self { - self.processed_messages.push(ProcessedMessageMark { - consumer_name: consumer_name.into(), - message_id: message_id.into(), - }); - self - } - pub fn into_write_plan(self) -> Result { let mut mutations = self.mutations; mutations.sort_by(|left, right| { @@ -638,7 +572,7 @@ impl ReadModelWritePlanBuilder { .into_iter() .map(|staged| staged.mutation) .collect::>(); - let plan = ReadModelWritePlan::new(mutations, self.processed_messages); + let plan = ReadModelWritePlan::new(mutations); plan.validate()?; Ok(plan) } @@ -978,15 +912,6 @@ where Ok(self) } - pub fn mark_processed( - &mut self, - consumer_name: impl Into, - message_id: impl Into, - ) -> &mut Self { - self.writes.mark_processed(consumer_name, message_id); - self - } - pub fn into_write_plan(self) -> Result { self.writes.into_write_plan() } diff --git a/src/repository/async_repository.rs b/src/repository/async_repository.rs index 27386b328..9a6240588 100644 --- a/src/repository/async_repository.rs +++ b/src/repository/async_repository.rs @@ -106,12 +106,6 @@ pub trait AsyncReadModelWritePlanStore: Send + Sync { &self, plan: ReadModelWritePlan, ) -> impl Future> + Send + '_; - - fn is_processed_async<'a>( - &'a self, - consumer_name: &'a str, - message_id: &'a str, - ) -> impl Future> + Send + 'a; } /// Async primary-key relational read-model query contract. diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index f62666e95..b62dafa25 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -8,7 +8,7 @@ reason = "async trait impls return impl Future + Send to preserve public Send bounds" )] -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use std::future::Future; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -24,11 +24,11 @@ use crate::outbox_worker::{ }; use crate::read_model::{ column_name_for, key_fingerprint, validate_key, validate_row_values, ColumnDef, ColumnType, - DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ProcessedMessageMark, - ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, - ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, - ReadModelSchema, ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, - RowValue, RowValues, RowWriteMode, Versioned, + DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, + ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, + ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, + ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, RowValue, + RowValues, RowWriteMode, Versioned, }; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, @@ -296,13 +296,7 @@ impl AsyncTransactionalCommit for SqliteRepository { } for plan in batch.read_model_plans { - let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - if let Some(mark) = outcome.duplicate_message() { - return Err(RepositoryError::Model(format!( - "processed message already handled by consumer `{}`: `{}`", - mark.consumer_name, mark.message_id - ))); - } + apply_read_model_write_plan_in_tx(&mut tx, plan).await?; } for write in batch.snapshots { @@ -339,20 +333,10 @@ impl AsyncReadModelWritePlanStore for SqliteRepository { validate_sql_write_plan(&plan)?; let mut tx = begin_read_model_tx(&self.pool).await?; let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - if outcome.was_applied() { - commit_read_model_tx(tx).await?; - } + commit_read_model_tx(tx).await?; Ok(outcome) } } - - fn is_processed_async<'a>( - &'a self, - consumer_name: &'a str, - message_id: &'a str, - ) -> impl Future> + Send + 'a { - async move { processed_message_exists_pool(&self.pool, consumer_name, message_id).await } - } } impl AsyncRelationalReadModelQueryStore for SqliteRepository { @@ -1252,7 +1236,6 @@ fn sql_read_model_capabilities() -> ReadModelAdapterCapabilities { relational_rows: true, sparse_patches: true, deletes: true, - processed_messages: true, } } @@ -1278,14 +1261,6 @@ async fn apply_read_model_write_plan_in_tx( ) -> Result { validate_sql_write_plan(&plan)?; - let mut marks_in_plan = HashSet::with_capacity(plan.processed_messages.len()); - for mark in &plan.processed_messages { - let key = processed_message_key(mark); - if !marks_in_plan.insert(key) || processed_message_exists_in_tx(tx, mark).await? { - return Ok(ReadModelCommitOutcome::skipped_duplicate(mark.clone())); - } - } - for mutation in plan.mutations { match mutation { ReadModelMutation::UpsertRow(mutation) => { @@ -1300,16 +1275,6 @@ async fn apply_read_model_write_plan_in_tx( } } - for mark in plan.processed_messages { - let result = insert_processed_message_in_tx(tx, &mark).await; - if let Err(err) = result { - if is_sqlite_unique_constraint(&err) { - return Ok(ReadModelCommitOutcome::skipped_duplicate(mark)); - } - return Err(read_model_storage_error("insert processed message", err)); - } - } - Ok(ReadModelCommitOutcome::applied()) } @@ -1908,66 +1873,6 @@ fn belongs_to_target_column( Ok(target_schema.primary_key.columns[0].clone()) } -async fn processed_message_exists_pool( - pool: &SqlitePool, - consumer_name: &str, - message_id: &str, -) -> Result { - let row = sqlx::query( - r#" - SELECT 1 - FROM read_model_processed_messages - WHERE consumer_name = ? AND message_id = ? - "#, - ) - .bind(consumer_name) - .bind(message_id) - .fetch_optional(pool) - .await - .map_err(|err| read_model_storage_error("load processed message", err))?; - Ok(row.is_some()) -} - -async fn processed_message_exists_in_tx( - tx: &mut Transaction<'_, Sqlite>, - mark: &ProcessedMessageMark, -) -> Result { - let row = sqlx::query( - r#" - SELECT 1 - FROM read_model_processed_messages - WHERE consumer_name = ? AND message_id = ? - "#, - ) - .bind(&mark.consumer_name) - .bind(&mark.message_id) - .fetch_optional(&mut **tx) - .await - .map_err(|err| read_model_storage_error("load processed message", err))?; - Ok(row.is_some()) -} - -async fn insert_processed_message_in_tx( - tx: &mut Transaction<'_, Sqlite>, - mark: &ProcessedMessageMark, -) -> Result<(), sqlx::Error> { - sqlx::query( - r#" - INSERT INTO read_model_processed_messages (consumer_name, message_id) - VALUES (?, ?) - "#, - ) - .bind(&mark.consumer_name) - .bind(&mark.message_id) - .execute(&mut **tx) - .await?; - Ok(()) -} - -fn processed_message_key(mark: &ProcessedMessageMark) -> (String, String) { - (mark.consumer_name.clone(), mark.message_id.clone()) -} - fn initial_row_version() -> u64 { 1 } diff --git a/tests/async_repository/main.rs b/tests/async_repository/main.rs index 09467a85c..3e7e18082 100644 --- a/tests/async_repository/main.rs +++ b/tests/async_repository/main.rs @@ -3,11 +3,11 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use sourced_rust::{ sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, - AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, - AsyncStreamWrite, AsyncTransactionalCommit, ClaimOutboxMessages, Entity, HashMapRepository, - InMemorySnapshotStore, OutboxMessage, ProcessedMessageMark, ReadModel, ReadModelWritePlan, - ReadModelWritePlanBuilder, RelationalReadModel, RepositoryError, RowKey, RowValue, - SnapshotRecord, Snapshottable, StreamIdentity, Versioned, + AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, ClaimOutboxMessages, Entity, HashMapRepository, + InMemorySnapshotStore, OutboxMessage, ReadModel, ReadModelWritePlanBuilder, + RelationalReadModel, RepositoryError, RowKey, RowValue, SnapshotRecord, Snapshottable, + StreamIdentity, Versioned, }; #[derive(Default)] @@ -133,36 +133,6 @@ async fn async_batch_rejects_duplicate_stream_identity_before_write() { assert!(repo.get_stream(&identity).await.unwrap().is_none()); } -#[tokio::test] -async fn async_batch_read_model_failure_rolls_back_stream_append() { - let repo = HashMapRepository::new(); - let identity = StreamIdentity::new("async.rollback", "rollback-1").unwrap(); - let mut entity = Entity::with_id("rollback-1"); - entity.digest_empty("Touched").unwrap(); - let mark = ProcessedMessageMark { - consumer_name: "projection".into(), - message_id: "event-1".into(), - }; - let plan = ReadModelWritePlan::new(Vec::new(), vec![mark.clone(), mark]); - - let err = repo - .commit_batch_async(AsyncCommitBatch { - streams: vec![AsyncStreamWrite::new(identity.clone(), &mut entity)], - outbox_messages: Vec::new(), - read_model_plans: vec![plan], - snapshots: Vec::new(), - }) - .await - .unwrap_err(); - - assert!( - matches!(err, RepositoryError::Model(message) if message.contains("processed message already handled")) - ); - assert!(repo.get_stream(&identity).await.unwrap().is_none()); - assert_eq!(entity.committed_version(), 0); - assert_eq!(entity.new_events().len(), 1); -} - #[tokio::test] async fn read_model_session_can_commit_against_async_store() { let repo = HashMapRepository::new(); @@ -171,21 +141,13 @@ async fn read_model_session_can_commit_against_async_store() { value: 42, }; let mut session = ReadModelWritePlanBuilder::new(); - session - .upsert(&view) - .unwrap() - .mark_processed("projection", "event-1"); + session.upsert(&view).unwrap(); let outcome = session.commit_async(&repo).await.unwrap(); let loaded = load_test_view(&repo, "view-1").await.unwrap(); - let processed = repo - .is_processed_async("projection", "event-1") - .await - .unwrap(); assert!(outcome.was_applied()); assert_eq!(loaded.data, view); - assert!(processed); } #[tokio::test] diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 750459715..292fcae8d 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -41,7 +41,7 @@ use checkout::{ SEAT_AVAILABLE, }; use checkout_saga_service::CheckoutSaga; -use projection_service::{service as projection_service, CHECKOUT_SCREEN_CONSUMER}; +use projection_service::service as projection_service; use query_service::CheckoutQueryService; use read_models::{register_schemas, CheckoutView}; #[cfg(any(feature = "sqlite", feature = "postgres"))] @@ -54,7 +54,7 @@ use sourced_rust::microsvc::{self, Service, Session}; use sourced_rust::SqliteRepository; use sourced_rust::{ AggregateBuilder, HashMapRepository, InMemoryQueue, InMemoryReadModelStore, OutboxWorkerThread, - Queueable, ReadModelWritePlanStore, + Queueable, }; #[cfg(any(feature = "sqlite", feature = "postgres"))] use sourced_rust::{ @@ -219,22 +219,6 @@ async fn run_async_persistent_checkout_flow( .expect("seat aggregate should exist"); assert_eq!(loaded_seat.status, SEAT_RESERVED); assert_eq!(loaded_seat.checkout_id, ids.checkout_id); - - for message in [ - &seat_added, - &checkout_started, - &seat_reserved, - &reservation_completed, - ] { - assert!( - read_repo - .is_processed_async(CHECKOUT_SCREEN_CONSUMER, message.id()) - .await - .expect("processed lookup should succeed"), - "message {} should be marked processed", - message.id() - ); - } } #[cfg(any(feature = "sqlite", feature = "postgres"))] @@ -455,7 +439,6 @@ where other => panic!("unexpected projected event type {other}"), } - read_models.mark_processed(CHECKOUT_SCREEN_CONSUMER, message.id()); read_models .commit_async(repo) .await @@ -620,18 +603,6 @@ fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { assert_eq!(seat.status, SEAT_RESERVED); assert_eq!(seat.checkout_id, "checkout-1"); - for event in queue.events() { - if projection_service::projects(event.event_type.as_str()) { - assert!( - read_store - .is_processed(CHECKOUT_SCREEN_CONSUMER, &event.id) - .expect("processed lookup should succeed"), - "event {} should be marked processed before ack", - event.id - ); - } - } - let _ = checkout_sub.stop(); let _ = seat_sub.stop(); let _ = projection_sub.stop(); diff --git a/tests/distributed_read_model/projection_service/handlers/checkout.rs b/tests/distributed_read_model/projection_service/handlers/checkout.rs index b8edfcb75..96574a07c 100644 --- a/tests/distributed_read_model/projection_service/handlers/checkout.rs +++ b/tests/distributed_read_model/projection_service/handlers/checkout.rs @@ -6,7 +6,7 @@ use crate::checkout::{ checkout_event, CheckoutStarted, SeatReservationCompleted, CHECKOUT_SEAT_RESERVED, CHECKOUT_STARTED, RESERVING_SEAT_MESSAGE, SEAT_RESERVED_MESSAGE, }; -use crate::projection_service::{ProjectionDependencies, CHECKOUT_SCREEN_CONSUMER}; +use crate::projection_service::ProjectionDependencies; use crate::read_models::{CheckoutStepView, CheckoutView}; pub const EVENTS: &[&str] = &[ @@ -42,7 +42,6 @@ pub fn handle(ctx: &Context) -> Result { @@ -69,7 +68,6 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), diff --git a/tests/distributed_read_model/projection_service/handlers/seat.rs b/tests/distributed_read_model/projection_service/handlers/seat.rs index b83979396..015c71edf 100644 --- a/tests/distributed_read_model/projection_service/handlers/seat.rs +++ b/tests/distributed_read_model/projection_service/handlers/seat.rs @@ -3,7 +3,7 @@ use sourced_rust::microsvc::{Context, HandlerError}; use sourced_rust::ReadModelWorkspaceExt; use crate::checkout::{seat_event, SeatAdded, SeatReserved, SEAT_AVAILABLE, SEAT_RESERVED}; -use crate::projection_service::{ProjectionDependencies, CHECKOUT_SCREEN_CONSUMER}; +use crate::projection_service::ProjectionDependencies; use crate::read_models::{CheckoutStepView, SeatView}; pub const EVENTS: &[&str] = &[seat_event::ADDED, seat_event::RESERVED]; @@ -29,7 +29,6 @@ pub fn handle(ctx: &Context) -> Result { @@ -51,7 +50,6 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), diff --git a/tests/distributed_read_model/projection_service/mod.rs b/tests/distributed_read_model/projection_service/mod.rs index 7b2c6fe65..20282db51 100644 --- a/tests/distributed_read_model/projection_service/mod.rs +++ b/tests/distributed_read_model/projection_service/mod.rs @@ -2,6 +2,4 @@ mod service; pub mod handlers; -pub use service::{projects, service, ProjectionDependencies}; - -pub const CHECKOUT_SCREEN_CONSUMER: &str = "checkout-screen-projection"; +pub use service::{service, ProjectionDependencies}; diff --git a/tests/distributed_read_model/projection_service/service.rs b/tests/distributed_read_model/projection_service/service.rs index 964c084be..bf967be12 100644 --- a/tests/distributed_read_model/projection_service/service.rs +++ b/tests/distributed_read_model/projection_service/service.rs @@ -14,7 +14,3 @@ pub fn service(store: InMemoryReadModelStore) -> Arc bool { - handlers::checkout::EVENTS.contains(&event_type) || handlers::seat::EVENTS.contains(&event_type) -} diff --git a/tests/distributed_read_model_board/main.rs b/tests/distributed_read_model_board/main.rs index bb72d1242..92b115731 100644 --- a/tests/distributed_read_model_board/main.rs +++ b/tests/distributed_read_model_board/main.rs @@ -18,14 +18,14 @@ use std::thread; use std::time::{Duration, Instant}; use board_service::{AddCard, MoveCard, OpenBoard, RemoveCard}; -use projections_service::{start_board_projection_service, wait_for_board, BOARD_CONSUMER}; +use projections_service::{start_board_projection_service, wait_for_board}; use query_service::BoardQueryService; use read_models::register_schemas; use serde::Serialize; use sourced_rust::microsvc::{Service, Session}; use sourced_rust::{ AggregateBuilder, HashMapRepository, InMemoryQueue, InMemoryReadModelStore, OutboxWorkerThread, - Queueable, ReadModelWritePlanStore, + Queueable, }; fn dispatch(service: &Service, command: &str, input: C) @@ -164,17 +164,6 @@ fn board_service_feeds_a_normalized_card_read_model() { .expect("query should succeed") .is_none()); - // Idempotency: every published event marked processed before ack. - for event in queue.events() { - assert!( - read_store - .is_processed(BOARD_CONSUMER, &event.id) - .expect("processed lookup should succeed"), - "event {} should be marked processed", - event.id - ); - } - let write_side = board_service .repo() .peek("board-1") diff --git a/tests/distributed_read_model_board/projections_service/handlers/board.rs b/tests/distributed_read_model_board/projections_service/handlers/board.rs index 02cac1c15..266419355 100644 --- a/tests/distributed_read_model_board/projections_service/handlers/board.rs +++ b/tests/distributed_read_model_board/projections_service/handlers/board.rs @@ -12,7 +12,6 @@ use crate::board_service::BoardSnapshot; use crate::projections_service::{read_model_error, ProjectionDependencies}; use crate::read_models::{board_key, BoardView, CardPayload, CardView}; -pub const CONSUMER: &str = "board-detail-projection"; pub const EVENTS: &[&str] = &[ "board.opened", "board.card_added", @@ -52,7 +51,6 @@ pub fn handle(ctx: &Context) -> Result(repo: R) +pub async fn standalone_relational_write_plan_persists_row(repo: R) where R: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore + Send + Sync, { @@ -49,12 +49,10 @@ where id: unique_id("read-model-view"), status: "available".into(), }; - let message_id = unique_id("read-model-message"); let mut read_models = ReadModelWritePlanBuilder::new(); read_models .upsert(&view) - .expect("row mutation should serialize") - .mark_processed("seat-projection", &message_id); + .expect("row mutation should serialize"); let outcome = read_models .commit_async(&repo) @@ -67,30 +65,6 @@ where .expect("read model row should exist"); assert_eq!(loaded.version, 1); assert_eq!(loaded.data, view); - assert!(repo - .is_processed_async("seat-projection", &message_id) - .await - .expect("processed marker should load")); - - let mut duplicate = ReadModelWritePlanBuilder::new(); - duplicate - .upsert(&SeatView { - id: view.id.clone(), - status: "reserved".into(), - }) - .expect("duplicate row mutation should serialize") - .mark_processed("seat-projection", &message_id); - - let duplicate_outcome = duplicate - .commit_async(&repo) - .await - .expect("duplicate processed message should be handled"); - let still_loaded = load_seat_view(&repo, &view.id) - .await - .expect("read model row should still exist"); - - assert!(duplicate_outcome.was_skipped()); - assert_eq!(still_loaded.data.status, "available"); } pub async fn aggregate_commit_persists_read_model_plan(repo: R) diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index 9be61f8d9..dc9435883 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -10,10 +10,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use sourced_rust::{ sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, - AsyncReadModelWritePlanCommitExt, AsyncReadModelWritePlanStore, AsyncSnapshotStore, - AsyncStreamWrite, AsyncTransactionalCommit, Entity, OutboxMessageStatus, PostgresRepository, - ReadModel, ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, - SnapshotRecord, StreamIdentity, TableSchemaRegistry, + AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, OutboxMessageStatus, PostgresRepository, ReadModel, + ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, + StreamIdentity, TableSchemaRegistry, }; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -154,10 +154,7 @@ async fn migration_is_idempotent_and_uses_postgres_column_types() { .fetch_optional(repo.pool()) .await .unwrap(); - assert_eq!( - processed_messages_table.as_deref(), - Some("read_model_processed_messages") - ); + assert!(processed_messages_table.is_none()); } #[tokio::test] @@ -396,29 +393,21 @@ async fn read_model_session_patches_and_deletes_relational_rows() { } #[tokio::test] -async fn read_model_session_persists_relational_rows_and_processed_marks() { +async fn read_model_session_persists_relational_rows() { let Some((_schema, repo)) = repository().await else { return; }; bootstrap_relational_counter_table(&repo).await; let id = unique_id("view"); - let message_id = unique_id("event"); let view = RelationalCounterView { id: id.clone(), value: 42, counts: HashMap::new(), }; let mut session = ReadModelWritePlanBuilder::new(); - session - .upsert(&view) - .unwrap() - .mark_processed("projection", &message_id); + session.upsert(&view).unwrap(); let outcome = session.commit_async(&repo).await.unwrap(); - let processed = repo - .is_processed_async("projection", &message_id) - .await - .unwrap(); let row = sqlx::query( r#" SELECT "value", "_sourced_version" @@ -437,36 +426,6 @@ async fn read_model_session_persists_relational_rows_and_processed_marks() { sqlx::Row::try_get::(&row, "_sourced_version").unwrap(), 1 ); - assert!(processed); - - let mut duplicate = ReadModelWritePlanBuilder::new(); - duplicate - .upsert(&RelationalCounterView { - id: id.clone(), - value: 100, - counts: HashMap::new(), - }) - .unwrap() - .mark_processed("projection", &message_id); - let duplicate_outcome = duplicate.commit_async(&repo).await.unwrap(); - let row = sqlx::query( - r#" - SELECT "value", "_sourced_version" - FROM "postgres_relational_counter_views" - WHERE "id" = $1 - "#, - ) - .bind(&view.id) - .fetch_one(repo.pool()) - .await - .unwrap(); - - assert!(duplicate_outcome.was_skipped()); - assert_eq!(sqlx::Row::try_get::(&row, "value").unwrap(), 42); - assert_eq!( - sqlx::Row::try_get::(&row, "_sourced_version").unwrap(), - 1 - ); } #[tokio::test] diff --git a/tests/read_model_distributed_idempotency/main.rs b/tests/read_model_distributed_idempotency/main.rs deleted file mode 100644 index 2056c4ea1..000000000 --- a/tests/read_model_distributed_idempotency/main.rs +++ /dev/null @@ -1,166 +0,0 @@ -use serde::{Deserialize, Serialize}; -use sourced_rust::bus::{Event, Publisher, Subscriber}; -use sourced_rust::{ - InMemoryQueue, InMemoryReadModelStore, ReadModel, ReadModelError, ReadModelWorkspaceExt, - ReadModelWritePlanBuilder, ReadModelWritePlanStore, RowKey, RowValue, -}; - -const CONSUMER: &str = "counter-projection"; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ReadModel)] -#[table("relational_counters")] -struct RelationalCounter { - #[id] - id: String, - value: i32, -} - -fn counter_session(view: &RelationalCounter, message_id: &str) -> ReadModelWritePlanBuilder { - let mut session = ReadModelWritePlanBuilder::new(); - session - .upsert(view) - .unwrap() - .mark_processed(CONSUMER, message_id); - session -} - -fn relational_counter_key(id: &str) -> RowKey { - RowKey::new([("id", RowValue::String(id.into()))]) -} - -fn load_counter( - store: &InMemoryReadModelStore, - id: &str, -) -> Option> { - store.register_schema::().unwrap(); - store - .workspace() - .load::(relational_counter_key(id)) - .one() - .unwrap() -} - -#[test] -fn standalone_session_commit_applies_row_and_marks_processed() { - let store = InMemoryReadModelStore::new(); - assert!(store.read_model_capabilities().processed_messages); - let view = RelationalCounter { - id: "counter-1".into(), - value: 1, - }; - - let outcome = counter_session(&view, "message-1").commit(&store).unwrap(); - - assert!(outcome.was_applied()); - assert!(store.is_processed(CONSUMER, "message-1").unwrap()); - let loaded = load_counter(&store, "counter-1").unwrap(); - assert_eq!(loaded.data, view); - assert_eq!(loaded.version, 1); -} - -#[test] -fn duplicate_processed_message_skips_mutations_idempotently() { - let store = InMemoryReadModelStore::new(); - let original = RelationalCounter { - id: "counter-1".into(), - value: 1, - }; - counter_session(&original, "message-1") - .commit(&store) - .unwrap(); - - let duplicate_update = RelationalCounter { - id: "counter-1".into(), - value: 99, - }; - let outcome = counter_session(&duplicate_update, "message-1") - .commit(&store) - .unwrap(); - - assert!(outcome.was_skipped()); - assert_eq!( - outcome - .duplicate_message() - .map(|mark| mark.message_id.as_str()), - Some("message-1") - ); - let loaded = load_counter(&store, "counter-1").unwrap(); - assert_eq!(loaded.data, original); - assert_eq!(loaded.version, 1); -} - -#[test] -fn read_model_write_and_processed_mark_are_atomic() { - let store = InMemoryReadModelStore::new(); - let row = RelationalCounter { - id: "counter-1".into(), - value: 1, - }; - let mut session = ReadModelWritePlanBuilder::new(); - session - .mark_processed(CONSUMER, "message-1") - .expect_version::(relational_counter_key("counter-1"), 99) - .unwrap() - .upsert(&row) - .unwrap(); - - let err = session.commit(&store).unwrap_err(); - - assert!(matches!(err, ReadModelError::NotFound { .. })); - assert!(!store.is_processed(CONSUMER, "message-1").unwrap()); - assert!(load_counter(&store, "counter-1").is_none()); -} - -#[test] -fn ack_happens_only_after_successful_standalone_commit() { - let queue = InMemoryQueue::new(); - let store = InMemoryReadModelStore::new(); - - queue - .publish(Event::with_string_payload( - "message-fail", - "CounterChanged", - "{}", - )) - .unwrap(); - let failed = queue.poll(0).unwrap().unwrap(); - let row = RelationalCounter { - id: "counter-1".into(), - value: 1, - }; - let mut failed_session = ReadModelWritePlanBuilder::new(); - failed_session - .expect_version::(relational_counter_key("counter-1"), 99) - .unwrap() - .upsert(&row) - .unwrap() - .mark_processed(CONSUMER, &failed.id); - - let err = failed_session.commit(&store).unwrap_err(); - - assert!(matches!(err, ReadModelError::NotFound { .. })); - assert!(queue.acknowledged().is_empty()); - assert!(!store.is_processed(CONSUMER, &failed.id).unwrap()); - - queue - .publish(Event::with_string_payload( - "message-ok", - "CounterChanged", - "{}", - )) - .unwrap(); - let succeeded = queue.poll(0).unwrap().unwrap(); - let view = RelationalCounter { - id: "counter-1".into(), - value: 2, - }; - let outcome = counter_session(&view, &succeeded.id) - .commit(&store) - .unwrap(); - assert!(outcome.was_applied()); - - queue.ack(&succeeded.id).unwrap(); - - assert_eq!(queue.acknowledged(), vec!["message-ok"]); - assert!(store.is_processed(CONSUMER, &succeeded.id).unwrap()); -} diff --git a/tests/read_model_relationship_includes/main.rs b/tests/read_model_relationship_includes/main.rs index edb5476ee..80496fb3d 100644 --- a/tests/read_model_relationship_includes/main.rs +++ b/tests/read_model_relationship_includes/main.rs @@ -88,10 +88,6 @@ impl ReadModelWritePlanStore for NoIncludeStore { ) -> Result { self.inner.commit_write_plan(plan) } - - fn is_processed(&self, consumer_name: &str, message_id: &str) -> Result { - self.inner.is_processed(consumer_name, message_id) - } } impl RelationalReadModelQueryStore for NoIncludeStore { diff --git a/tests/read_model_session/main.rs b/tests/read_model_session/main.rs index 805b83db4..696f6796d 100644 --- a/tests/read_model_session/main.rs +++ b/tests/read_model_session/main.rs @@ -302,7 +302,7 @@ fn relationship_operation_populates_child_foreign_key_in_explicit_row_mutation() } #[test] -fn expected_versions_and_processed_messages_are_carried_into_plan() { +fn expected_versions_are_carried_into_plan() { let mut account = AccountSummary::new("acct-1"); let loaded = Versioned { data: account.clone(), @@ -315,8 +315,7 @@ fn expected_versions_and_processed_messages_are_carried_into_plan() { .track_loaded(&loaded) .unwrap() .upsert(&account) - .unwrap() - .mark_processed("account-projection", "message-1"); + .unwrap(); let plan = session.into_write_plan().unwrap(); @@ -324,11 +323,6 @@ fn expected_versions_and_processed_messages_are_carried_into_plan() { panic!("expected upsert row"); }; assert_eq!(row.expected_version, ExpectedVersion::Exact(7)); - assert_eq!( - plan.processed_messages[0].consumer_name, - "account-projection" - ); - assert_eq!(plan.processed_messages[0].message_id, "message-1"); } #[test] diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 25b704675..bd45306f9 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -5,10 +5,10 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use sourced_rust::{ sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, - AsyncReadModelWritePlanCommitExt, AsyncReadModelWritePlanStore, AsyncSnapshotStore, - AsyncStreamWrite, AsyncTransactionalCommit, Entity, OutboxMessageStatus, ReadModel, - ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, - SqliteRepository, StreamIdentity, TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, + AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, OutboxMessageStatus, ReadModel, ReadModelWritePlanBuilder, + RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, SqliteRepository, StreamIdentity, + TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, }; #[derive(Default)] @@ -323,7 +323,7 @@ async fn read_model_session_patches_and_deletes_relational_rows() { } #[tokio::test] -async fn read_model_session_persists_relational_rows_and_processed_marks() { +async fn read_model_session_persists_relational_rows() { let repo = repository().await; bootstrap_relational_counter_table(&repo).await; let view = RelationalCounterView { @@ -332,16 +332,9 @@ async fn read_model_session_persists_relational_rows_and_processed_marks() { counts: HashMap::new(), }; let mut session = ReadModelWritePlanBuilder::new(); - session - .upsert(&view) - .unwrap() - .mark_processed("projection", "event-1"); + session.upsert(&view).unwrap(); let outcome = session.commit_async(&repo).await.unwrap(); - let processed = repo - .is_processed_async("projection", "event-1") - .await - .unwrap(); let row = sqlx::query( r#" SELECT "value", "_sourced_version" @@ -360,36 +353,6 @@ async fn read_model_session_persists_relational_rows_and_processed_marks() { sqlx::Row::try_get::(&row, "_sourced_version").unwrap(), 1 ); - assert!(processed); - - let mut duplicate = ReadModelWritePlanBuilder::new(); - duplicate - .upsert(&RelationalCounterView { - id: "view-1".into(), - value: 100, - counts: HashMap::new(), - }) - .unwrap() - .mark_processed("projection", "event-1"); - let duplicate_outcome = duplicate.commit_async(&repo).await.unwrap(); - let row = sqlx::query( - r#" - SELECT "value", "_sourced_version" - FROM "local_relational_counter_views" - WHERE "id" = ? - "#, - ) - .bind("view-1") - .fetch_one(repo.pool()) - .await - .unwrap(); - - assert!(duplicate_outcome.was_skipped()); - assert_eq!(sqlx::Row::try_get::(&row, "value").unwrap(), 42); - assert_eq!( - sqlx::Row::try_get::(&row, "_sourced_version").unwrap(), - 1 - ); } #[tokio::test] From 9f0dfc1ba69e0cc84981a0c34326308893f9821e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 20:07:51 -0500 Subject: [PATCH 02/39] feat(transport): async transport foundation in microsvc::transport Establishes the shared async transport layer in three reviewed slices: - Core contracts: TransportError (retryable/permanent), FailurePolicy/FailureAction, RunOptions/ConsumerDeliveryMode/InboxHook, TransportCapabilities, stable-id rules. - Source runner: AsyncMessageSource/ReceivedMessage + run_source (ack after handler success, retryable->nack, permanent->failure policy, ack-and-ignore unhandled, graceful stop, no swallowed errors). - Publisher/outbox bridge: AsyncMessagePublisher, OutboxMessage->Message mapping (reserved x-sourced- metadata namespace), OutboxDispatcher (dispatch_ids/ dispatch_batch sharing claim->publish->complete), claim-by-id across HashMap/SQLite/Postgres, From for TransportError. Not feature-gated; executor-agnostic (no tokio dependency). Verified with cargo test --all-features (242 lib unit tests + integration/conformance suites), fmt, clippy, and doc-link checks. Implements [[tasks/async-transport-core-contracts]], [[tasks/async-message-source-runner]], and [[tasks/async-message-publisher-outbox]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/mod.rs | 1 + src/microsvc/transport/capabilities.rs | 248 ++++++++ src/microsvc/transport/error.rs | 268 +++++++++ src/microsvc/transport/failure_policy.rs | 143 +++++ src/microsvc/transport/mod.rs | 99 ++++ src/microsvc/transport/outbox_dispatch.rs | 488 ++++++++++++++++ src/microsvc/transport/publisher.rs | 125 ++++ src/microsvc/transport/run_options.rs | 230 ++++++++ src/microsvc/transport/runner.rs | 552 ++++++++++++++++++ src/microsvc/transport/source.rs | 82 +++ src/microsvc/transport/stable_id.rs | 141 +++++ src/outbox_worker/store.rs | 92 +++ src/postgres_repo/mod.rs | 2 + src/sqlite_repo/mod.rs | 80 ++- tests/hashmap_repository_conformance/main.rs | 10 + .../outbox.rs | 71 +++ tests/postgres_repository_conformance/main.rs | 12 + tests/sqlite_repository_conformance/main.rs | 10 + 18 files changed, 2622 insertions(+), 32 deletions(-) create mode 100644 src/microsvc/transport/capabilities.rs create mode 100644 src/microsvc/transport/error.rs create mode 100644 src/microsvc/transport/failure_policy.rs create mode 100644 src/microsvc/transport/mod.rs create mode 100644 src/microsvc/transport/outbox_dispatch.rs create mode 100644 src/microsvc/transport/publisher.rs create mode 100644 src/microsvc/transport/run_options.rs create mode 100644 src/microsvc/transport/runner.rs create mode 100644 src/microsvc/transport/source.rs create mode 100644 src/microsvc/transport/stable_id.rs diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index b691a3faf..265120493 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -58,6 +58,7 @@ mod dependencies; mod error; mod service; mod session; +pub mod transport; pub use context::Context; pub use dependencies::{ diff --git a/src/microsvc/transport/capabilities.rs b/src/microsvc/transport/capabilities.rs new file mode 100644 index 000000000..069a3d320 --- /dev/null +++ b/src/microsvc/transport/capabilities.rs @@ -0,0 +1,248 @@ +//! Transport capability metadata. +//! +//! Different transports place receive durability, publish confirmation, retry +//! ownership, acknowledgement, and Knative integration in different places. +//! [`TransportCapabilities`] makes those differences explicit so docs, tests, +//! and adapter selection can reason about them instead of assuming every +//! transport behaves like one reference broker. +//! +//! The named constructors encode the comparison matrix from +//! `specs/transport-interface-evaluation`. + +/// How a consumer acknowledges successful handler execution back to its +/// transport. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum ConsumerAckKind { + /// Complete / delete / archive a durable table row (Postgres). + TableRow, + /// Ack / nack a broker delivery (RabbitMQ / servicebus AMQP). + DeliveryAck, + /// Commit the consumer group offset (Kafka). + OffsetCommit, + /// Ack / nak / term a stream message (NATS JetStream). + StreamAck, + /// Return a successful HTTP response (Knative / CloudEvents). + HttpResponse, + /// In-process acknowledgement only (in-memory dev/test transport). + InProcess, +} + +/// How a transport integrates with Knative Eventing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum KnativeIntegrationKind { + /// First-class Knative Eventing path (Kafka and RabbitMQ Broker/Source, + /// Knative HTTP CloudEvents). + Native, + /// Needs a custom bridge that POSTs CloudEvents to a Knative Broker or is + /// triggered by Knative and republishes to the transport (Postgres, NATS + /// JetStream). + CustomBridge, + /// No Knative integration (in-memory dev/test transport). + None, +} + +/// Capability profile for a transport adapter. +/// +/// The two confirmation thresholds are distinct and both matter: +/// +/// - `publish_confirm` is the *producer* threshold an outbox row waits for +/// before it is marked published. +/// - `consumer_ack` is the *consumer* threshold the runner waits for before the +/// adapter acknowledges receipt. +/// +/// `platform_managed_retry` distinguishes platform-driven delivery (Knative +/// owns retry, backoff, and dead-lettering) from direct transports where the +/// adapter and this crate own that operational contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct TransportCapabilities { + /// The transport durably stores received messages until acknowledged (a + /// table, queue, stream, or broker), rather than relying on a live process + /// to hold them. + pub durable_receive: bool, + /// Publishing has a durable confirmation threshold (transaction commit, + /// publisher confirm, producer ack, or a successful HTTP response). + pub publish_confirm: bool, + /// Retry, backoff, and dead-lettering are managed by the platform rather + /// than by this crate and the adapter. + pub platform_managed_retry: bool, + /// How the consumer acknowledges successful execution. + pub consumer_ack: ConsumerAckKind, + /// How the transport integrates with Knative Eventing. + pub knative_integration: KnativeIntegrationKind, +} + +impl TransportCapabilities { + /// Postgres: durable table-backed transport, transaction-commit publish + /// confirmation, app-owned retry, row-completion ack, custom Knative bridge. + pub const fn postgres() -> Self { + Self { + durable_receive: true, + publish_confirm: true, + platform_managed_retry: false, + consumer_ack: ConsumerAckKind::TableRow, + knative_integration: KnativeIntegrationKind::CustomBridge, + } + } + + /// RabbitMQ / servicebus: durable broker, publisher-confirm publish, app/ + /// broker-owned retry topology, delivery-ack consumer, native Knative path. + pub const fn rabbitmq() -> Self { + Self { + durable_receive: true, + publish_confirm: true, + platform_managed_retry: false, + consumer_ack: ConsumerAckKind::DeliveryAck, + knative_integration: KnativeIntegrationKind::Native, + } + } + + /// Kafka: durable log, producer-ack publish, app-owned retry/DLQ topics, + /// offset-commit ack, native Knative path. + pub const fn kafka() -> Self { + Self { + durable_receive: true, + publish_confirm: true, + platform_managed_retry: false, + consumer_ack: ConsumerAckKind::OffsetCommit, + knative_integration: KnativeIntegrationKind::Native, + } + } + + /// NATS JetStream: durable stream, publish-ack publish, JetStream/custom + /// retry, stream-ack consumer, custom Knative bridge. + pub const fn nats_jetstream() -> Self { + Self { + durable_receive: true, + publish_confirm: true, + platform_managed_retry: false, + consumer_ack: ConsumerAckKind::StreamAck, + knative_integration: KnativeIntegrationKind::CustomBridge, + } + } + + /// Knative / HTTP CloudEvents: platform-delivered (no app-owned durable + /// receive store), HTTP-response publish confirmation, platform-managed + /// retry/backoff/dead-letter, HTTP-response ack, native integration. + pub const fn knative() -> Self { + Self { + durable_receive: false, + publish_confirm: true, + platform_managed_retry: true, + consumer_ack: ConsumerAckKind::HttpResponse, + knative_integration: KnativeIntegrationKind::Native, + } + } + + /// In-memory dev/test transport: not durable, best-effort acceptance with no + /// durable publish confirmation, no platform retry, in-process ack, no + /// Knative integration. + pub const fn in_memory() -> Self { + Self { + durable_receive: false, + publish_confirm: false, + platform_managed_retry: false, + consumer_ack: ConsumerAckKind::InProcess, + knative_integration: KnativeIntegrationKind::None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn postgres_is_durable_with_custom_knative_bridge() { + let caps = TransportCapabilities::postgres(); + assert!(caps.durable_receive); + assert!(caps.publish_confirm); + assert!(!caps.platform_managed_retry); + assert_eq!(caps.consumer_ack, ConsumerAckKind::TableRow); + assert_eq!( + caps.knative_integration, + KnativeIntegrationKind::CustomBridge + ); + } + + #[test] + fn knative_delegates_retry_to_the_platform() { + let caps = TransportCapabilities::knative(); + assert!(caps.platform_managed_retry); + assert!(!caps.durable_receive); + assert_eq!(caps.consumer_ack, ConsumerAckKind::HttpResponse); + assert_eq!(caps.knative_integration, KnativeIntegrationKind::Native); + } + + #[test] + fn in_memory_is_the_only_profile_without_durable_receive_or_publish_confirm() { + let caps = TransportCapabilities::in_memory(); + assert!(!caps.durable_receive); + assert!(!caps.publish_confirm); + assert_eq!(caps.consumer_ack, ConsumerAckKind::InProcess); + assert_eq!(caps.knative_integration, KnativeIntegrationKind::None); + } + + #[test] + fn direct_transports_own_their_retry() { + for caps in [ + TransportCapabilities::postgres(), + TransportCapabilities::rabbitmq(), + TransportCapabilities::kafka(), + TransportCapabilities::nats_jetstream(), + ] { + assert!( + !caps.platform_managed_retry, + "direct transports own retry; only Knative is platform-managed" + ); + assert!(caps.durable_receive, "direct transports receive durably"); + } + } + + #[test] + fn each_known_transport_has_a_distinct_ack_kind() { + let acks = [ + TransportCapabilities::postgres().consumer_ack, + TransportCapabilities::rabbitmq().consumer_ack, + TransportCapabilities::kafka().consumer_ack, + TransportCapabilities::nats_jetstream().consumer_ack, + TransportCapabilities::knative().consumer_ack, + TransportCapabilities::in_memory().consumer_ack, + ]; + for (i, a) in acks.iter().enumerate() { + for b in &acks[i + 1..] { + assert_ne!(a, b, "ack kinds should be distinct across transports"); + } + } + } + + #[test] + fn knative_integration_matches_the_evaluation_matrix() { + use KnativeIntegrationKind::*; + // Native Eventing paths. + assert_eq!( + TransportCapabilities::rabbitmq().knative_integration, + Native + ); + assert_eq!(TransportCapabilities::kafka().knative_integration, Native); + assert_eq!(TransportCapabilities::knative().knative_integration, Native); + // Custom-bridge transports. + assert_eq!( + TransportCapabilities::postgres().knative_integration, + CustomBridge + ); + assert_eq!( + TransportCapabilities::nats_jetstream().knative_integration, + CustomBridge + ); + // No integration. + assert_eq!(TransportCapabilities::in_memory().knative_integration, None); + } + + #[test] + fn capabilities_round_trip_through_serde() { + let caps = TransportCapabilities::kafka(); + let json = serde_json::to_string(&caps).unwrap(); + let restored: TransportCapabilities = serde_json::from_str(&json).unwrap(); + assert_eq!(caps, restored); + } +} diff --git a/src/microsvc/transport/error.rs b/src/microsvc/transport/error.rs new file mode 100644 index 000000000..ca6a9eb9e --- /dev/null +++ b/src/microsvc/transport/error.rs @@ -0,0 +1,268 @@ +//! Transport error classification. +//! +//! Async transport adapters and the runner that drives them need a shared way +//! to say whether a failure is worth retrying. [`TransportError`] carries that +//! classification so the runner can decide between negative-acknowledging a +//! message for redelivery (retryable) and handing it to the configured +//! [`FailurePolicy`](super::FailurePolicy) (permanent). + +use std::error::Error; +use std::fmt; + +use crate::microsvc::HandlerError; + +/// Whether a [`TransportError`] should be retried. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum TransportErrorKind { + /// A transient failure. The message should be redelivered and retried. + /// + /// Unknown outcomes (for example, a publish whose acknowledgement was lost) + /// are retryable: duplicate delivery is acceptable under at-least-once + /// semantics, but silently dropping the work is not. + Retryable, + /// A deterministic failure. Retrying the same message as-is will not help, + /// so the runner consults the failure policy (dead-letter, park, + /// log-and-ack, or stop) instead of redelivering forever. + Permanent, +} + +impl TransportErrorKind { + /// Whether this kind is retryable. + pub fn is_retryable(self) -> bool { + matches!(self, TransportErrorKind::Retryable) + } + + /// Whether this kind is permanent. + pub fn is_permanent(self) -> bool { + matches!(self, TransportErrorKind::Permanent) + } +} + +/// An error raised by a transport adapter or runner, classified as retryable or +/// permanent. +/// +/// The classification is the contract the runner relies on; the message and +/// optional source are for diagnostics, logging, and dead-letter metadata. +#[derive(Debug)] +pub struct TransportError { + kind: TransportErrorKind, + message: String, + source: Option>, +} + +impl TransportError { + /// Create a retryable transport error. + pub fn retryable(message: impl Into) -> Self { + Self { + kind: TransportErrorKind::Retryable, + message: message.into(), + source: None, + } + } + + /// Create a permanent transport error. + pub fn permanent(message: impl Into) -> Self { + Self { + kind: TransportErrorKind::Permanent, + message: message.into(), + source: None, + } + } + + /// Create a transport error with an explicit classification. + pub fn new(kind: TransportErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + source: None, + } + } + + /// Attach an underlying source error for diagnostics. + pub fn with_source(mut self, source: impl Error + Send + Sync + 'static) -> Self { + self.source = Some(Box::new(source)); + self + } + + /// The retry classification of this error. + pub fn kind(&self) -> TransportErrorKind { + self.kind + } + + /// Whether this error is retryable. + pub fn is_retryable(&self) -> bool { + self.kind.is_retryable() + } + + /// Whether this error is permanent. + pub fn is_permanent(&self) -> bool { + self.kind.is_permanent() + } + + /// The human-readable message, without the classification prefix. + pub fn message(&self) -> &str { + &self.message + } + + /// Classify a [`HandlerError`] for transport retry purposes. + /// + /// Transient failures are retryable: repository errors, not-found results, + /// and otherwise-unclassified errors. Not-found is transient in an + /// at-least-once event-driven system because an out-of-order delivery can + /// reference an aggregate or projection that has not been created yet, and a + /// later redelivery resolves the race. (This also keeps a handler-raised + /// [`HandlerError::NotFound`] consistent with a repository-raised + /// [`RepositoryError::NotFound`](crate::RepositoryError), which arrives + /// wrapped in `Repository(_)`.) + /// + /// Deterministic failures — unknown routing, payload decode failures, + /// business rejections, authorization, and guard rejections — are permanent + /// because redelivering the identical message cannot change the outcome. An + /// entity that is genuinely missing forever still drains to the failure + /// policy once the transport's retry ceiling is reached, so defaulting + /// not-found to retryable does not risk an unbounded loop. + /// + /// This is a default. Handlers that know better can return a classification + /// directly, and the runner's [`FailurePolicy`](super::FailurePolicy) can + /// override what happens to a permanent error. + pub fn classify_handler_error(error: &HandlerError) -> TransportErrorKind { + match error { + HandlerError::Repository(_) | HandlerError::NotFound(_) | HandlerError::Other(_) => { + TransportErrorKind::Retryable + } + HandlerError::UnknownCommand(_) + | HandlerError::DecodeFailed(_) + | HandlerError::Rejected(_) + | HandlerError::Unauthorized(_) + | HandlerError::GuardRejected(_) => TransportErrorKind::Permanent, + } + } +} + +impl fmt::Display for TransportError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let kind = match self.kind { + TransportErrorKind::Retryable => "retryable", + TransportErrorKind::Permanent => "permanent", + }; + write!(f, "transport error ({kind}): {}", self.message) + } +} + +impl Error for TransportError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.source + .as_ref() + .map(|source| source.as_ref() as &(dyn Error + 'static)) + } +} + +impl From for TransportError { + fn from(error: HandlerError) -> Self { + let kind = Self::classify_handler_error(&error); + Self { + kind, + message: error.to_string(), + source: Some(Box::new(error)), + } + } +} + +impl From for TransportError { + /// Repository/store failures (lock contention, storage hiccups, stale-claim + /// conflicts) are treated as retryable: they are usually transient, and a + /// later redelivery or re-claim resolves them. + fn from(error: crate::RepositoryError) -> Self { + Self { + kind: TransportErrorKind::Retryable, + message: error.to_string(), + source: Some(Box::new(error)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retryable_and_permanent_classify_themselves() { + let retry = TransportError::retryable("connection reset"); + assert!(retry.is_retryable()); + assert!(!retry.is_permanent()); + assert_eq!(retry.kind(), TransportErrorKind::Retryable); + + let permanent = TransportError::permanent("bad payload"); + assert!(permanent.is_permanent()); + assert!(!permanent.is_retryable()); + assert_eq!(permanent.kind(), TransportErrorKind::Permanent); + } + + #[test] + fn display_includes_classification_and_message() { + assert_eq!( + TransportError::retryable("lease lost").to_string(), + "transport error (retryable): lease lost" + ); + assert_eq!( + TransportError::permanent("decode failed").to_string(), + "transport error (permanent): decode failed" + ); + } + + #[test] + fn with_source_is_exposed_through_error_trait() { + let inner = std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out"); + let err = TransportError::retryable("publish timed out").with_source(inner); + assert!(err.source().is_some()); + assert_eq!(err.message(), "publish timed out"); + } + + #[test] + fn transient_handler_errors_are_retryable() { + use crate::repository::RepositoryError; + + for error in [ + HandlerError::Repository(RepositoryError::NotFound { id: "agg-1".into() }), + // A handler-raised not-found is transient too: it is usually an + // out-of-order delivery race, and stays consistent with the + // repository-raised not-found above. + HandlerError::NotFound("agg-1".into()), + HandlerError::Other("boom".into()), + ] { + assert_eq!( + TransportError::classify_handler_error(&error), + TransportErrorKind::Retryable, + "{error} should be retryable" + ); + } + } + + #[test] + fn deterministic_handler_errors_are_permanent() { + for error in [ + HandlerError::UnknownCommand("x".into()), + HandlerError::DecodeFailed("x".into()), + HandlerError::Rejected("x".into()), + HandlerError::Unauthorized("x".into()), + HandlerError::GuardRejected("x".into()), + ] { + assert_eq!( + TransportError::classify_handler_error(&error), + TransportErrorKind::Permanent, + "{error} should be permanent" + ); + } + } + + #[test] + fn from_handler_error_preserves_classification_and_source() { + let err: TransportError = HandlerError::Rejected("invalid".into()).into(); + assert!(err.is_permanent()); + assert!(err.source().is_some()); + + let err: TransportError = + HandlerError::Other(Box::::from("infra")).into(); + assert!(err.is_retryable()); + } +} diff --git a/src/microsvc/transport/failure_policy.rs b/src/microsvc/transport/failure_policy.rs new file mode 100644 index 000000000..a1a740729 --- /dev/null +++ b/src/microsvc/transport/failure_policy.rs @@ -0,0 +1,143 @@ +//! Failure handling policy for the async transport runner. +//! +//! The runner classifies each handler/transport failure with +//! [`TransportError`](super::TransportError). Retryable failures are always +//! redelivered. For *permanent* failures the runner consults a +//! [`FailurePolicy`] and performs the resolved [`FailureAction`]. The default +//! never silently acknowledges a handler error. + +use super::TransportError; + +/// What the runner should do with a message after a failure. +/// +/// This is the decision; the runner (a later subtask) performs the side effect +/// through the transport adapter's ack/nack/dead-letter primitives. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum FailureAction { + /// Negative-acknowledge (or leave unacked) so the transport redelivers the + /// message later, subject to its retry/backoff/max-attempt rules. + Nack, + /// Route the message to a dead-letter destination, then acknowledge it so it + /// leaves the main flow. + DeadLetter, + /// Hold the message for manual intervention without acknowledging it and + /// without automatic redelivery. Adapters that cannot park should surface a + /// clear error rather than silently dropping the message. + Park, + /// Log the error and acknowledge the message, dropping it. This is the only + /// action that discards a failed message, so it must be opt-in. + LogAndAck, + /// Stop the runner loop and surface the error to the caller. + Stop, +} + +/// Policy for handling *permanent* (non-retryable) failures. +/// +/// Retryable failures bypass the policy and are always nacked for redelivery; +/// the policy decides what happens once an error is terminal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum FailurePolicy { + /// Redeliver everything, even permanent failures. Bounded only by the + /// transport's retry/backoff/max-attempt configuration. + Retry, + /// Send permanent failures to a dead-letter destination, then acknowledge. + DeadLetter, + /// Park permanent failures for manual intervention. + Park, + /// Log permanent failures and acknowledge, dropping the message. + LogAndAck, + /// Stop the runner on a permanent failure. + Stop, +} + +impl Default for FailurePolicy { + /// Dead-letter: a permanent failure is moved aside for inspection rather + /// than dropped or redelivered forever. Adapters without a dead-letter + /// destination should treat this as [`FailureAction::Park`] and report a + /// clear error instead of silently acknowledging. + fn default() -> Self { + FailurePolicy::DeadLetter + } +} + +impl FailurePolicy { + /// Resolve the action the runner should take for a classified error. + /// + /// Retryable errors always resolve to [`FailureAction::Nack`] regardless of + /// policy, because "retry later" is the correct response to a transient + /// failure on every transport. Only permanent errors are routed through the + /// configured policy. + pub fn resolve(self, error: &TransportError) -> FailureAction { + if error.is_retryable() { + return FailureAction::Nack; + } + match self { + FailurePolicy::Retry => FailureAction::Nack, + FailurePolicy::DeadLetter => FailureAction::DeadLetter, + FailurePolicy::Park => FailureAction::Park, + FailurePolicy::LogAndAck => FailureAction::LogAndAck, + FailurePolicy::Stop => FailureAction::Stop, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_policy_dead_letters_permanent_failures() { + assert_eq!(FailurePolicy::default(), FailurePolicy::DeadLetter); + let action = FailurePolicy::default().resolve(&TransportError::permanent("bad")); + assert_eq!(action, FailureAction::DeadLetter); + } + + #[test] + fn retryable_errors_always_nack_regardless_of_policy() { + let retry = TransportError::retryable("transient"); + for policy in [ + FailurePolicy::Retry, + FailurePolicy::DeadLetter, + FailurePolicy::Park, + FailurePolicy::LogAndAck, + FailurePolicy::Stop, + ] { + assert_eq!( + policy.resolve(&retry), + FailureAction::Nack, + "{policy:?} should nack a retryable error" + ); + } + } + + #[test] + fn permanent_failures_follow_the_configured_policy() { + let permanent = TransportError::permanent("terminal"); + assert_eq!( + FailurePolicy::Retry.resolve(&permanent), + FailureAction::Nack + ); + assert_eq!( + FailurePolicy::DeadLetter.resolve(&permanent), + FailureAction::DeadLetter + ); + assert_eq!(FailurePolicy::Park.resolve(&permanent), FailureAction::Park); + assert_eq!( + FailurePolicy::LogAndAck.resolve(&permanent), + FailureAction::LogAndAck + ); + assert_eq!(FailurePolicy::Stop.resolve(&permanent), FailureAction::Stop); + } + + #[test] + fn only_log_and_ack_discards_a_failed_message() { + let permanent = TransportError::permanent("terminal"); + let discards = + |policy: FailurePolicy| policy.resolve(&permanent) == FailureAction::LogAndAck; + assert!(discards(FailurePolicy::LogAndAck)); + assert!(!discards(FailurePolicy::Retry)); + assert!(!discards(FailurePolicy::DeadLetter)); + assert!(!discards(FailurePolicy::Park)); + assert!(!discards(FailurePolicy::Stop)); + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs new file mode 100644 index 000000000..fa299908e --- /dev/null +++ b/src/microsvc/transport/mod.rs @@ -0,0 +1,99 @@ +//! Shared vocabulary for async message transports. +//! +//! `microsvc` owns handler registration, guards, typed input decoding, and +//! dispatch. *Transport adapters* own how messages are received, acknowledged, +//! retried, published, and mapped to external topics/subjects/routes/triggers. +//! This module is the neutral vocabulary both sides share — it does not +//! implement any concrete broker. +//! +//! The direct-transport receive path lives here too: +//! +//! - [`AsyncMessageSource`] / [`ReceivedMessage`] — pull a message from a direct +//! transport (Postgres, RabbitMQ, Kafka, NATS, in-memory) and settle it; +//! - [`run_source`] — the runner that dispatches through +//! `Service::dispatch_message`, then acks/nacks/dead-letters per policy. +//! +//! The producer-side publish path lives here too: +//! +//! - [`AsyncMessagePublisher`] — the single publish boundary, with each adapter's +//! durable publish threshold documented; +//! - [`OutboxDispatcher`] / [`OutboxDispatchOutcome`] — map durable outbox rows to +//! `Message` and dispatch them, sharing one claim → publish → complete path +//! between background polling and after-commit immediate dispatch. +//! +//! The Knative/HTTP ingress shape and the concrete broker adapters are still +//! separate slices. Everything builds on the vocabulary defined here: +//! +//! - [`TransportError`] / [`TransportErrorKind`] — retryable vs permanent +//! classification the runner uses to decide between redelivery and the +//! failure policy. +//! - [`FailurePolicy`] / [`FailureAction`] — what happens to a permanent +//! failure: dead-letter, park, log-and-ack, retry, or stop. +//! - [`RunOptions`] / [`ConsumerDeliveryMode`] / [`InboxHook`] — idempotent +//! dispatch by default, with a placeholder hook for the future consumer +//! inbox. +//! - [`TransportCapabilities`] — how each transport differs in receive +//! durability, publish confirmation, retry ownership, acknowledgement, and +//! Knative integration. +//! - [`validate_stable_message_id`] — the rules an inbox-enabled run uses to +//! reject messages that lack a usable deduplication key. +//! +//! # Two confirmation thresholds +//! +//! Producing and consuming have *separate* completion thresholds, and they must +//! not be conflated: +//! +//! **Producer publish threshold** — when an outbox row may be marked published. +//! Only after the adapter's durable publish confirmation: +//! +//! - Postgres: the outbox-backed bus row committed, or a committed insert into a +//! separate queue table; +//! - RabbitMQ: publisher confirm; +//! - Kafka: the producer send acknowledged per the configured `acks`; +//! - NATS JetStream: a JetStream publish ack; +//! - Knative / HTTP: a successful response from the Broker/sink; +//! - in-memory: accepted into the in-memory queue/log. +//! +//! If the publish outcome is unknown, the outbox row stays retryable. Duplicate +//! delivery is acceptable under at-least-once semantics. +//! +//! **Consumer ack threshold** — when the adapter may acknowledge receipt. Only +//! after the runner reports successful consumer execution: +//! +//! - the guard passed (or the message was intentionally ignored by routing); +//! - the handler returned success; +//! - the handler's aggregate / read-model / outbox writes committed; +//! - in inbox mode, the inbox receipt committed atomically with those effects. +//! +//! How that acknowledgement maps back to the transport is adapter-owned and +//! described by [`ConsumerAckKind`]: a row completion, a delivery ack, an offset +//! commit, a stream ack, or a 2xx HTTP response. The default never silently +//! acknowledges a handler error — retryable failures redeliver and permanent +//! failures go through the [`FailurePolicy`]. +//! +//! Producer-side immediate dispatch is *not* a transport acknowledgement: it is +//! best-effort delivery after the local transaction commits. Consumer-side +//! deduplication, when needed, is the optional consumer inbox, not an outbox or +//! publish guarantee. + +use crate::microsvc::Message; + +mod capabilities; +mod error; +mod failure_policy; +mod outbox_dispatch; +mod publisher; +mod run_options; +mod runner; +mod source; +mod stable_id; + +pub use capabilities::{ConsumerAckKind, KnativeIntegrationKind, TransportCapabilities}; +pub use error::{TransportError, TransportErrorKind}; +pub use failure_policy::{FailureAction, FailurePolicy}; +pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; +pub use publisher::AsyncMessagePublisher; +pub use run_options::{ConsumerDeliveryMode, InboxHook, NoInbox, RunOptions}; +pub use runner::run_source; +pub use source::{AsyncMessageSource, ReceivedMessage}; +pub use stable_id::{validate_stable_message_id, StableMessageIdError, MAX_STABLE_MESSAGE_ID_LEN}; diff --git a/src/microsvc/transport/outbox_dispatch.rs b/src/microsvc/transport/outbox_dispatch.rs new file mode 100644 index 000000000..c9c86b84f --- /dev/null +++ b/src/microsvc/transport/outbox_dispatch.rs @@ -0,0 +1,488 @@ +//! Outbox → transport bridge. +//! +//! Maps durable [`OutboxMessage`] rows to the canonical [`Message`] and dispatches +//! them through an [`AsyncMessagePublisher`]. The same claim → map → publish → +//! settle path is shared by background worker polling ([`dispatch_batch`]) and +//! after-commit immediate dispatch ([`dispatch_ids`]), so the two cannot diverge +//! and cannot publish the same row concurrently — both go through the outbox +//! claim lease. +//! +//! [`dispatch_batch`]: OutboxDispatcher::dispatch_batch +//! [`dispatch_ids`]: OutboxDispatcher::dispatch_ids + +use std::time::Duration; + +use super::publisher::AsyncMessagePublisher; +use super::TransportError; +use crate::microsvc::{Message, MessageKind}; +use crate::outbox::OutboxMessage; +use crate::outbox_worker::{ + AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, +}; + +/// Content type for an outbox payload. Outbox payloads are codec-encoded bytes +/// (bitcode or raw), so the media type is binary; the exact codec travels in +/// metadata under [`SOURCED_METADATA_PREFIX`] for the consumer. +const OUTBOX_CONTENT_TYPE: &str = "application/octet-stream"; + +/// Reserved metadata key prefix for framework-derived keys. User metadata must +/// not use this prefix; keys here (payload codec, destination, source context) +/// carry decode/routing semantics and must not be shadowable by user metadata. +pub const SOURCED_METADATA_PREFIX: &str = "x-sourced-"; + +impl From<&OutboxMessage> for Message { + /// Map a durable outbox row to a canonical transport message. + /// + /// - `id` ← outbox message id (the stable durable id); + /// - `name` ← `event_type`; + /// - `kind` ← `Command` when a point-to-point `destination` is set, else `Event`; + /// - `payload` ← raw codec bytes, `content_type` = `application/octet-stream`; + /// - `metadata` ← the outbox metadata (correlation/causation/trace/auth) plus + /// framework-derived keys under the reserved [`SOURCED_METADATA_PREFIX`] + /// namespace (payload codec, destination, source-aggregate context) so + /// decode/routing context can never be shadowed by a user metadata key. + fn from(outbox: &OutboxMessage) -> Self { + let kind = if outbox.destination.is_some() { + MessageKind::Command + } else { + MessageKind::Event + }; + + // User metadata first (correlation/trace/auth), then framework-derived + // keys under the reserved `x-sourced-` prefix. Any user key in the + // reserved namespace is dropped so framework values stay authoritative + // and cannot be shadowed on case-insensitive lookup. + let mut metadata: Vec<(String, String)> = outbox + .metadata + .iter() + .filter(|(key, _)| { + !key.to_ascii_lowercase() + .starts_with(SOURCED_METADATA_PREFIX) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}payload-codec"), + outbox.payload_codec.clone(), + )); + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}payload-codec-version"), + outbox.payload_codec_version.to_string(), + )); + if let Some(destination) = &outbox.destination { + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}destination"), + destination.clone(), + )); + } + if let Some(source_type) = &outbox.source_aggregate_type { + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}source-aggregate-type"), + source_type.clone(), + )); + } + if let Some(source_id) = &outbox.source_aggregate_id { + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}source-aggregate-id"), + source_id.clone(), + )); + } + if let Some(sequence) = outbox.source_sequence { + metadata.push(( + format!("{SOURCED_METADATA_PREFIX}source-sequence"), + sequence.to_string(), + )); + } + + Message { + id: Some(outbox.id().to_string()), + name: outbox.event_type.clone(), + kind, + payload: outbox.payload.clone(), + content_type: OUTBOX_CONTENT_TYPE.to_string(), + metadata, + } + } +} + +/// Counts of what one dispatch pass did. Raced/unclaimable ids are reflected as +/// `claimed < requested`, not as an error; publish failures are `released` +/// (retryable) or `failed` (attempt ceiling reached), not errors. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct OutboxDispatchOutcome { + /// Rows asked for (id count for `dispatch_ids`, batch size for `dispatch_batch`). + pub requested: usize, + /// Rows actually claimed this pass. + pub claimed: usize, + /// Rows published and completed. + pub published: usize, + /// Rows released for retry after a publish failure. + pub released: usize, + /// Rows permanently failed after exhausting attempts. + pub failed: usize, +} + +/// Bridges outbox claims to an [`AsyncMessagePublisher`], shared by immediate +/// after-commit dispatch and background worker polling. +/// +/// Publish failures are not errors — they are reflected in +/// [`OutboxDispatchOutcome`] (released/failed) and the pass continues. A *store* +/// error (claim/complete/record_failure) does abort the pass and is returned as +/// `Err`; the partial [`OutboxDispatchOutcome`] is not returned in that case, and +/// any already-settled rows keep their settled state (the next pass resumes the +/// rest). Rows still leased by the aborted pass become claimable again at lease +/// expiry. +pub struct OutboxDispatcher { + store: S, + publisher: P, + worker_id: String, + lease: Duration, + max_attempts: u32, +} + +impl OutboxDispatcher +where + S: AsyncOutboxStore, + P: AsyncMessagePublisher, +{ + /// Create a dispatcher. `worker_id` scopes claims (use a synthetic id such as + /// `immediate:` for after-commit dispatch); `max_attempts` is the + /// publish-failure ceiling before a row is permanently failed. + pub fn new( + store: S, + publisher: P, + worker_id: impl Into, + lease: Duration, + max_attempts: u32, + ) -> Self { + Self { + store, + publisher, + worker_id: worker_id.into(), + lease, + max_attempts, + } + } + + /// Immediate after-commit dispatch of the explicit outbox ids a commit just + /// inserted. Claims those ids (raced/unclaimable ids are skipped, not an + /// error) before publishing, so it never races the polling worker. + pub async fn dispatch_ids( + &self, + ids: &[String], + ) -> Result { + let request = + ClaimOutboxMessages::for_ids(self.worker_id.clone(), ids.to_vec(), self.lease); + let claimed = self.store.claim_async(request).await?; + let mut outcome = self.dispatch_claimed(claimed).await?; + outcome.requested = ids.len(); + Ok(outcome) + } + + /// Background worker dispatch of the next claimable batch. + pub async fn dispatch_batch( + &self, + batch_size: usize, + ) -> Result { + let request = ClaimOutboxMessages::new(self.worker_id.clone(), batch_size, self.lease); + let claimed = self.store.claim_async(request).await?; + let mut outcome = self.dispatch_claimed(claimed).await?; + outcome.requested = batch_size; + Ok(outcome) + } + + /// The shared settle path: map → publish → complete on success, or + /// record_failure (release/fail) on publish error. A row is completed ONLY + /// after the publish threshold; an unknown/failed publish leaves it retryable. + async fn dispatch_claimed( + &self, + claimed: Vec, + ) -> Result { + let mut outcome = OutboxDispatchOutcome { + claimed: claimed.len(), + ..Default::default() + }; + for message in claimed { + let claim = OutboxClaimRef::from_message(&message)?; + let transport_message = Message::from(&message); + match self.publisher.publish(transport_message).await { + Ok(()) => { + self.store.complete_async(&claim).await?; + outcome.published += 1; + } + Err(publish_error) => { + match self + .store + .record_failure_async(&claim, &publish_error.to_string(), self.max_attempts) + .await? + { + OutboxPublishFailureAction::Released => outcome.released += 1, + OutboxPublishFailureAction::Failed => outcome.failed += 1, + } + } + } + } + Ok(outcome) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{HashMapRepository, TransactionalCommit}; + use std::future::Future; + use std::sync::Mutex; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + /// Publisher that records every published message id, optionally failing. + struct RecordingPublisher { + published: Mutex>, + fail: bool, + } + + impl RecordingPublisher { + fn new(fail: bool) -> Self { + Self { + published: Mutex::new(Vec::new()), + fail, + } + } + fn ids(&self) -> Vec { + self.published.lock().unwrap().clone() + } + } + + impl AsyncMessagePublisher for RecordingPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + if self.fail { + // Unknown outcome: surface as a retryable error. + return Err(TransportError::retryable("publish failed")); + } + self.published + .lock() + .unwrap() + .push(message.id().unwrap_or_default().to_string()); + Ok(()) + } + } + + fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { + let id = message.id().to_string(); + let mut batch = crate::CommitBatch::empty(); + batch.outbox_messages.push(message); + repo.commit_batch(batch).unwrap(); + id + } + + fn outbox(id: &str) -> OutboxMessage { + OutboxMessage::create_with_metadata( + id, + "OrderCreated", + b"\x01\x02".to_vec(), + [("correlation_id".to_string(), "corr-1".to_string())] + .into_iter() + .collect(), + ) + .unwrap() + } + + #[test] + fn maps_outbox_row_to_canonical_message() { + let message = Message::from(&outbox("evt-1")); + assert_eq!(message.id(), Some("evt-1")); + assert_eq!(message.name(), "OrderCreated"); + assert_eq!(message.kind, MessageKind::Event); + assert_eq!(message.payload(), b"\x01\x02"); + assert_eq!(message.content_type, "application/octet-stream"); + // User metadata preserved; codec carried (namespaced) for the consumer. + assert_eq!(message.correlation_id(), Some("corr-1")); + assert_eq!(message.metadata("x-sourced-payload-codec"), Some("bytes")); + assert_eq!( + message.metadata("x-sourced-payload-codec-version"), + Some("1") + ); + } + + #[test] + fn user_metadata_cannot_shadow_reserved_framework_keys() { + // A user metadata key colliding with a reserved name does not override + // the framework value, because framework keys are namespaced. + let outbox = OutboxMessage::create_with_metadata( + "evt-1", + "OrderCreated", + b"\x01".to_vec(), + [("x-sourced-payload-codec".to_string(), "evil".to_string())] + .into_iter() + .collect(), + ) + .unwrap(); + let message = Message::from(&outbox); + // The user's reserved-prefix key is dropped; lookup returns the + // authoritative framework value, and there is exactly one such entry. + assert_eq!(message.metadata("x-sourced-payload-codec"), Some("bytes")); + assert_eq!( + message + .metadata + .iter() + .filter(|(k, _)| k == "x-sourced-payload-codec") + .count(), + 1 + ); + } + + #[test] + fn destination_maps_to_command_kind() { + let outbox = + OutboxMessage::create_to("cmd-1", "ShipOrder", "shipping", b"{}".to_vec()).unwrap(); + let message = Message::from(&outbox); + assert_eq!(message.kind, MessageKind::Command); + assert_eq!(message.metadata("x-sourced-destination"), Some("shipping")); + } + + fn dispatcher( + repo: &HashMapRepository, + fail: bool, + max_attempts: u32, + ) -> OutboxDispatcher { + OutboxDispatcher::new( + repo.outbox_store(), + RecordingPublisher::new(fail), + "immediate:test", + Duration::from_secs(60), + max_attempts, + ) + } + + fn load(repo: &HashMapRepository, id: &str) -> OutboxMessage { + repo.outbox_storage() + .read() + .unwrap() + .get(id) + .unwrap() + .clone() + } + + #[test] + fn dispatch_ids_claims_then_publishes_then_completes() { + let repo = HashMapRepository::new(); + let id = store_message(&repo, outbox("evt-1")); + let dispatcher = dispatcher(&repo, false, 3); + + let outcome = block_on(dispatcher.dispatch_ids(std::slice::from_ref(&id))).unwrap(); + + assert_eq!( + outcome, + OutboxDispatchOutcome { + requested: 1, + claimed: 1, + published: 1, + released: 0, + failed: 0, + } + ); + // The publisher saw the message id, and the row is completed only after. + assert_eq!(dispatcher.publisher.ids(), vec!["evt-1".to_string()]); + assert!(load(&repo, &id).is_published()); + } + + #[test] + fn unknown_publish_outcome_leaves_row_retryable() { + let repo = HashMapRepository::new(); + let id = store_message(&repo, outbox("evt-1")); + let dispatcher = dispatcher(&repo, true, 3); + + let outcome = block_on(dispatcher.dispatch_ids(std::slice::from_ref(&id))).unwrap(); + + assert_eq!(outcome.published, 0); + assert_eq!(outcome.released, 1); + assert_eq!(outcome.failed, 0); + // Released back to pending, not completed: still retryable. + let row = load(&repo, &id); + assert!(row.is_pending()); + assert_eq!(row.attempts, 1); + } + + #[test] + fn publish_failure_fails_row_at_attempt_ceiling() { + let repo = HashMapRepository::new(); + let id = store_message(&repo, outbox("evt-1")); + let dispatcher = dispatcher(&repo, true, 1); + + let outcome = block_on(dispatcher.dispatch_ids(std::slice::from_ref(&id))).unwrap(); + + assert_eq!(outcome.failed, 1); + assert_eq!(outcome.released, 0); + assert!(load(&repo, &id).is_failed()); + } + + #[test] + fn dispatch_ids_only_claims_requested_ids() { + let repo = HashMapRepository::new(); + let wanted = store_message(&repo, outbox("evt-1")); + let other = store_message(&repo, outbox("evt-2")); + let dispatcher = dispatcher(&repo, false, 3); + + let outcome = block_on(dispatcher.dispatch_ids(std::slice::from_ref(&wanted))).unwrap(); + + assert_eq!(outcome.claimed, 1); + assert_eq!(outcome.published, 1); + assert!(load(&repo, &wanted).is_published()); + // The unrequested row is untouched. + assert!(load(&repo, &other).is_pending()); + } + + #[test] + fn raced_id_is_not_an_error() { + let repo = HashMapRepository::new(); + // No such row stored: claim returns nothing, dispatch is a clean no-op. + let outcome = + block_on(dispatcher(&repo, false, 3).dispatch_ids(&["missing".to_string()])).unwrap(); + assert_eq!( + outcome, + OutboxDispatchOutcome { + requested: 1, + claimed: 0, + published: 0, + released: 0, + failed: 0, + } + ); + } + + #[test] + fn worker_and_immediate_dispatch_share_state_transitions() { + let repo = HashMapRepository::new(); + let immediate_id = store_message(&repo, outbox("evt-1")); + let _worker_id = store_message(&repo, outbox("evt-2")); + + // Immediate dispatch claims+completes evt-1; worker batch then drains the + // rest. Both go through the same claim/publish/complete path. + let dispatcher = dispatcher(&repo, false, 3); + let immediate = + block_on(dispatcher.dispatch_ids(std::slice::from_ref(&immediate_id))).unwrap(); + assert_eq!(immediate.published, 1); + + let drained = block_on(dispatcher.dispatch_batch(10)).unwrap(); + assert_eq!(drained.claimed, 1, "only evt-2 remains claimable"); + assert_eq!(drained.published, 1); + assert_eq!(dispatcher.publisher.ids().len(), 2); + } +} diff --git a/src/microsvc/transport/publisher.rs b/src/microsvc/transport/publisher.rs new file mode 100644 index 000000000..c4ac29bd6 --- /dev/null +++ b/src/microsvc/transport/publisher.rs @@ -0,0 +1,125 @@ +//! The shared async publish boundary. +//! +//! Producing is more uniform than consuming, so outbox dispatch and any other +//! producer uses a single [`AsyncMessagePublisher`]. Each adapter documents its +//! durable *publish threshold* — the point `publish` may resolve `Ok`: +//! +//! - Postgres: the outbox-backed bus row committed, or a committed insert into a +//! separate queue table; +//! - RabbitMQ: publisher confirm; +//! - Kafka: the producer send acknowledged per the configured `acks`; +//! - NATS JetStream: a JetStream publish ack; +//! - Knative / HTTP: a successful response from the Broker/sink; +//! - in-memory: accepted into the in-memory queue/log. +//! +//! Until that threshold, `publish` must not resolve `Ok`. An unknown outcome +//! must surface as `Err` so the outbox row stays retryable — duplicate delivery +//! is acceptable under at-least-once, silent loss is not. + +use std::future::Future; + +use super::{Message, TransportError}; + +/// Publishes canonical [`Message`]s to a transport. +/// +/// `publish` resolves `Ok` only once the adapter's durable publish threshold is +/// reached; any failure or unknown outcome is `Err`. The error's +/// [retryability](TransportError) lets the caller (the outbox dispatcher) decide +/// whether to keep the row retryable. +pub trait AsyncMessagePublisher: Send + Sync { + /// Publish a single message. + fn publish( + &self, + message: Message, + ) -> impl Future> + Send + '_; + + /// Publish a batch of messages. + /// + /// The default publishes sequentially and stops at the first error, so a + /// partial batch may have been published when this returns `Err`; the + /// caller settles each outbox row by its own claim, so partial progress is + /// safe. Adapters with native batching (a Kafka producer batch, a single + /// multi-row transaction) should override this. + #[allow(clippy::manual_async_fn)] + fn publish_batch( + &self, + messages: Vec, + ) -> impl Future> + Send + '_ { + async move { + for message in messages { + self.publish(message).await?; + } + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::MessageKind; + use std::future::Future; + use std::sync::Mutex; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + /// Publisher that records publishes and fails once it reaches `fail_at`. + struct CountingPublisher { + published: Mutex>, + fail_at: Option, + } + + impl AsyncMessagePublisher for CountingPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + let mut published = self.published.lock().unwrap(); + if self.fail_at == Some(published.len()) { + return Err(TransportError::retryable("publish failed")); + } + published.push(message.name().to_string()); + Ok(()) + } + } + + fn msg(name: &str) -> Message { + Message::new(name, MessageKind::Event, b"{}".to_vec()) + } + + #[test] + fn publish_batch_publishes_all_on_success() { + let publisher = CountingPublisher { + published: Mutex::new(Vec::new()), + fail_at: None, + }; + block_on(publisher.publish_batch(vec![msg("a"), msg("b"), msg("c")])).unwrap(); + assert_eq!(*publisher.published.lock().unwrap(), vec!["a", "b", "c"]); + } + + #[test] + fn publish_batch_stops_at_first_error_with_partial_progress() { + let publisher = CountingPublisher { + published: Mutex::new(Vec::new()), + fail_at: Some(1), // fail on the second message + }; + let result = block_on(publisher.publish_batch(vec![msg("a"), msg("b"), msg("c")])); + assert!(result.is_err()); + // Only the first message was published before the error. + assert_eq!(*publisher.published.lock().unwrap(), vec!["a"]); + } +} diff --git a/src/microsvc/transport/run_options.rs b/src/microsvc/transport/run_options.rs new file mode 100644 index 000000000..3ff1533d4 --- /dev/null +++ b/src/microsvc/transport/run_options.rs @@ -0,0 +1,230 @@ +//! Cross-cutting execution options for the async transport runner. +//! +//! [`RunOptions`] owns the policy that wraps `Service::dispatch_message`: +//! whether consumer execution is plain idempotent dispatch or wrapped by a +//! future consumer inbox, and what to do with permanent failures. Transports +//! stay free of this policy — they only obey the rule "acknowledge after the +//! runner reports success." + +use super::stable_id::{validate_stable_message_id, StableMessageIdError}; +use super::{FailurePolicy, Message}; + +/// Placeholder shape for the future consumer inbox hook. +/// +/// A consumer inbox (see `specs/consumer-inbox-design`) wraps handler execution +/// so a message receipt commits atomically with the handler's side effects, +/// giving effectively-once local database effects on top of at-least-once +/// delivery. This slice only defines the *shape* of that hook so [`RunOptions`] +/// can carry one; the storage model and execution wrapper land with the inbox +/// subtask. +/// +/// Every inbox scopes receipts by a consumer identity, which is the one piece +/// of contract that is already settled, so it is the only method defined here. +/// The trait is provisional: the inbox subtask is expected to add the +/// (asynchronous) execution-wrapping method, so external implementors should +/// expect it to grow rather than treat this shape as final. +pub trait InboxHook { + /// The stable consumer name receipts are scoped to. + fn consumer_name(&self) -> &str; +} + +/// How the runner executes consumers for a run. +/// +/// Defaults to [`ConsumerDeliveryMode::Idempotent`]: the convention is +/// idempotent handlers/projections, so a redelivered message is safe. +pub enum ConsumerDeliveryMode { + /// Dispatch directly and acknowledge after handler success. Handlers are + /// expected to be idempotent under redelivery. + Idempotent, + /// Wrap dispatch with a consumer inbox; acknowledge only after the inbox + /// receipt and handler side effects commit together. Requires a stable + /// message id (see [`RunOptions::validate_message_id`]). + Inbox(I), +} + +// A `#[derive(Default)]` would add an `I: Default` bound, which would forbid +// `ConsumerDeliveryMode` (NoInbox is uninhabited). The default variant +// carries no `I`, so the bound is spurious — implement it by hand instead. +#[allow(clippy::derivable_impls)] +impl Default for ConsumerDeliveryMode { + fn default() -> Self { + ConsumerDeliveryMode::Idempotent + } +} + +/// Uninhabited marker for run options that carry no inbox hook. +/// +/// Because it can never be constructed, a `RunOptions` is statically +/// guaranteed to be in idempotent mode — which is exactly the default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NoInbox {} + +/// Cross-cutting execution policy for a transport run. +/// +/// Generic over the inbox hook type `I`, defaulting to [`NoInbox`] for the +/// common idempotent case. +pub struct RunOptions { + /// Whether consumer execution is plain idempotent dispatch or inbox-wrapped. + pub delivery_mode: ConsumerDeliveryMode, + /// What the runner does with a permanent handler/transport failure. + pub failure_policy: FailurePolicy, +} + +impl Default for RunOptions { + /// Idempotent delivery with the default [`FailurePolicy`]. + fn default() -> Self { + Self { + delivery_mode: ConsumerDeliveryMode::default(), + failure_policy: FailurePolicy::default(), + } + } +} + +impl RunOptions { + /// Idempotent run options: direct dispatch, default failure policy, no + /// stable-id requirement. + pub fn idempotent() -> Self { + Self::default() + } +} + +impl RunOptions { + /// Inbox-wrapped run options. Acknowledgement waits for the inbox receipt + /// and handler side effects to commit, and a stable message id becomes + /// required. + pub fn inbox(hook: I) -> Self { + Self { + delivery_mode: ConsumerDeliveryMode::Inbox(hook), + failure_policy: FailurePolicy::default(), + } + } + + /// Override the failure policy. + pub fn with_failure_policy(mut self, policy: FailurePolicy) -> Self { + self.failure_policy = policy; + self + } + + /// Whether this run dispatches directly without an inbox. + pub fn is_idempotent(&self) -> bool { + matches!(self.delivery_mode, ConsumerDeliveryMode::Idempotent) + } + + /// Whether this run requires a stable message id. True in inbox mode, + /// because deduplication needs a durable key. + pub fn requires_stable_id(&self) -> bool { + matches!(self.delivery_mode, ConsumerDeliveryMode::Inbox(_)) + } + + /// Validate a message's id against this run's requirements, returning the + /// dedup key when one is required. + /// + /// Inbox runs enforce the [stable id rules](validate_stable_message_id) and + /// return `Ok(Some(id))` — the borrowed id the inbox should key its receipt + /// on, exactly as the transport delivered it. A failure here is a permanent + /// condition for the message: redelivery cannot supply a missing or + /// malformed id. Idempotent runs need no dedup key and return `Ok(None)`, + /// accepting any id including none. + pub fn validate_message_id<'m>( + &self, + message: &'m Message, + ) -> Result, StableMessageIdError> { + if self.requires_stable_id() { + validate_stable_message_id(message.id()).map(Some) + } else { + Ok(None) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::MessageKind; + + struct FakeInbox { + consumer: &'static str, + } + + impl InboxHook for FakeInbox { + fn consumer_name(&self) -> &str { + self.consumer + } + } + + fn message_with_id(id: Option<&str>) -> Message { + let mut message = Message::new("seat.reserved", MessageKind::Event, b"{}".to_vec()); + if let Some(id) = id { + message = message.with_id(id); + } + message + } + + #[test] + fn default_run_options_are_idempotent_with_default_policy() { + let options = RunOptions::::default(); + assert!(options.is_idempotent()); + assert!(!options.requires_stable_id()); + assert_eq!(options.failure_policy, FailurePolicy::default()); + } + + #[test] + fn idempotent_constructor_matches_default() { + let options = RunOptions::idempotent(); + assert!(options.is_idempotent()); + assert!(!options.requires_stable_id()); + } + + #[test] + fn inbox_mode_requires_a_stable_id_and_keeps_the_hook() { + let options = RunOptions::inbox(FakeInbox { + consumer: "seat-projection", + }); + assert!(!options.is_idempotent()); + assert!(options.requires_stable_id()); + match &options.delivery_mode { + ConsumerDeliveryMode::Inbox(hook) => { + assert_eq!(hook.consumer_name(), "seat-projection") + } + ConsumerDeliveryMode::Idempotent => panic!("expected inbox mode"), + } + } + + #[test] + fn with_failure_policy_overrides_the_default() { + let options = RunOptions::idempotent().with_failure_policy(FailurePolicy::Stop); + assert_eq!(options.failure_policy, FailurePolicy::Stop); + } + + #[test] + fn idempotent_mode_needs_no_dedup_key() { + let options = RunOptions::idempotent(); + // No key is required, so even a present id yields `Ok(None)`. + assert_eq!( + options.validate_message_id(&message_with_id(None)), + Ok(None) + ); + assert_eq!( + options.validate_message_id(&message_with_id(Some("evt-1"))), + Ok(None) + ); + } + + #[test] + fn inbox_mode_returns_the_validated_dedup_key() { + let options = RunOptions::inbox(FakeInbox { consumer: "c" }); + assert_eq!( + options.validate_message_id(&message_with_id(None)), + Err(StableMessageIdError::Missing) + ); + assert_eq!( + options.validate_message_id(&message_with_id(Some(" "))), + Err(StableMessageIdError::Empty) + ); + // A valid id is returned for the inbox to key its receipt on. + assert_eq!( + options.validate_message_id(&message_with_id(Some("evt-1"))), + Ok(Some("evt-1")) + ); + } +} diff --git a/src/microsvc/transport/runner.rs b/src/microsvc/transport/runner.rs new file mode 100644 index 000000000..55612385b --- /dev/null +++ b/src/microsvc/transport/runner.rs @@ -0,0 +1,552 @@ +//! The direct-source runner. +//! +//! [`run_source`] is the shared receive loop for direct transports. It owns the +//! cross-cutting policy — when execution counts as successful, how retryable vs +//! permanent failures are routed — while the adapter owns how acknowledgement +//! maps back to the transport. The same dispatch/runner boundary is what the +//! Knative/HTTP ingress will call, so consumer execution stays identical across +//! ingress shapes. + +use std::sync::Arc; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{FailureAction, RunOptions, TransportError}; +use crate::microsvc::{Message, Service}; + +/// Run the receive loop for a direct transport source. +/// +/// For each message the runner: +/// +/// 1. enforces the inbox stable-id contract (a no-op in idempotent mode); +/// 2. dispatches through [`Service::dispatch_message`]; +/// 3. on success, acknowledges via the adapter; +/// 4. on failure, routes through [`RunOptions::failure_policy`] — retryable +/// failures are nacked for redelivery, permanent failures take the configured +/// action ([dead-letter](FailureAction::DeadLetter), [park](FailureAction::Park), +/// [log-and-ack](FailureAction::LogAndAck), or [stop](FailureAction::Stop)). +/// +/// A message with no registered handler is **intentionally ignored**: the runner +/// acks it and moves on rather than dead-lettering it. Fan-out event transports +/// may deliver events this service does not consume, and acking matches +/// `microsvc::subscribe`; production transports should use +/// [`Service::subscription_plan`] to avoid delivering unrelated messages at all. +/// +/// The runner **acks only after handler effects have completed**, never before. +/// It stops gracefully when the source returns `Ok(None)`, having fully settled +/// the in-flight message first. Receive and settle errors are propagated, not +/// swallowed: a returned `Err` ends the run and the supervisor may restart it +/// (already-committed effects make redelivery safe). +/// +/// Inbox note: until the consumer-inbox subtask lands, inbox mode enforces the +/// stable-id requirement and then dispatches like idempotent mode. The +/// receipt-commit wrapping that makes it effectively-once is added there. +/// +/// `I: Send` keeps the returned future `Send` so the runner can be spawned on a +/// multi-threaded executor regardless of the inbox hook type. +pub async fn run_source( + service: Arc>, + mut source: S, + options: RunOptions, +) -> Result<(), TransportError> +where + D: Send + Sync + 'static, + S: AsyncMessageSource, + I: Send, +{ + while let Some(received) = source.recv().await? { + // No handler for this message: intentionally ignore (ack) rather than + // dead-letter, so unrelated fan-out events don't pile into the DLQ. + if !service.handles_message(received.message().kind, received.message().name()) { + received.ack().await?; + continue; + } + match dispatch(&service, &options, received.message()) { + Ok(()) => received.ack().await?, + Err(error) => match options.failure_policy.resolve(&error) { + FailureAction::Nack => received.nack(&error.to_string()).await?, + FailureAction::DeadLetter => received.dead_letter(&error.to_string()).await?, + FailureAction::Park => received.park(&error.to_string()).await?, + FailureAction::LogAndAck => { + eprintln!( + "[microsvc::transport] dropping message '{}' after permanent failure: {error}", + received.message().name() + ); + received.ack().await? + } + FailureAction::Stop => return Err(error), + }, + } + } + Ok(()) +} + +/// Run consumer execution for one message and classify the outcome. +/// +/// Enforces the inbox stable-id contract first (idempotent mode yields no key +/// and skips it), then dispatches. A failed stable-id check is a permanent +/// failure — redelivery cannot supply a missing or malformed id. +fn dispatch( + service: &Service, + options: &RunOptions, + message: &Message, +) -> Result<(), TransportError> +where + D: Send + Sync + 'static, +{ + options + .validate_message_id(message) + .map_err(|err| TransportError::permanent(err.to_string()).with_source(err))?; + service + .dispatch_message(message) + .map(|_| ()) + .map_err(TransportError::from) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::transport::FailurePolicy; + use crate::microsvc::{HandlerError, MessageKind}; + use serde_json::json; + use std::collections::VecDeque; + use std::future::Future; + use std::sync::Mutex; + + // --- minimal runtime-free executor ------------------------------------- + // The transport module is not feature-gated, so its tests run without an + // async runtime. The fake futures never suspend, so a busy-poll with a + // no-op waker drives them to completion. + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let raw = RawWaker::new(ptr::null(), &VTABLE); + let waker = unsafe { Waker::from_raw(raw) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + // --- recorder + fakes --------------------------------------------------- + #[derive(Debug, Clone, PartialEq, Eq)] + enum Event { + Handled(String), + Ack, + Nack(String), + DeadLetter(String), + Park(String), + } + + struct Recorder { + events: Mutex>, + } + + impl Recorder { + fn new() -> Arc { + Arc::new(Self { + events: Mutex::new(Vec::new()), + }) + } + fn push(&self, event: Event) { + self.events.lock().unwrap().push(event); + } + fn events(&self) -> Vec { + self.events.lock().unwrap().clone() + } + } + + struct FakeReceived { + message: Message, + recorder: Arc, + settle_ok: bool, + } + + impl FakeReceived { + fn settle(self, event: Event) -> Result<(), TransportError> { + self.recorder.push(event); + if self.settle_ok { + Ok(()) + } else { + Err(TransportError::retryable("settle failed")) + } + } + } + + impl ReceivedMessage for FakeReceived { + fn message(&self) -> &Message { + &self.message + } + async fn ack(self) -> Result<(), TransportError> { + self.settle(Event::Ack) + } + async fn nack(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::Nack(reason.to_string())) + } + async fn dead_letter(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::DeadLetter(reason.to_string())) + } + async fn park(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::Park(reason.to_string())) + } + } + + struct FakeSource { + queue: VecDeque, + recorder: Arc, + settle_ok: bool, + recv_error: bool, + } + + impl AsyncMessageSource for FakeSource { + type Received = FakeReceived; + async fn recv(&mut self) -> Result, TransportError> { + if self.recv_error { + return Err(TransportError::retryable("recv failed")); + } + Ok(self.queue.pop_front().map(|message| FakeReceived { + message, + recorder: self.recorder.clone(), + settle_ok: self.settle_ok, + })) + } + } + + // --- helpers ------------------------------------------------------------ + fn event_message(name: &str, id: Option<&str>) -> Message { + let mut message = Message::new(name, MessageKind::Event, b"{}".to_vec()); + if let Some(id) = id { + message = message.with_id(id); + } + message + } + + fn service(recorder: &Arc) -> Arc> { + let ok = recorder.clone(); + let retryable = recorder.clone(); + let permanent = recorder.clone(); + Arc::new( + Service::new(()) + .event("ok") + .handle(move |ctx| { + ok.push(Event::Handled(ctx.message().name().to_string())); + Ok(json!({})) + }) + .event("retryable") + .handle(move |ctx| { + retryable.push(Event::Handled(ctx.message().name().to_string())); + Err(HandlerError::Other("infra".into())) + }) + .event("permanent") + .handle(move |ctx| { + permanent.push(Event::Handled(ctx.message().name().to_string())); + Err(HandlerError::Rejected("nope".into())) + }), + ) + } + + struct RunResult { + outcome: Result<(), TransportError>, + events: Vec, + } + + fn run(messages: Vec, options: RunOptions) -> RunResult { + run_with(messages, options, true, false) + } + + fn run_with( + messages: Vec, + options: RunOptions, + settle_ok: bool, + recv_error: bool, + ) -> RunResult { + let recorder = Recorder::new(); + let svc = service(&recorder); + let source = FakeSource { + queue: messages.into_iter().collect(), + recorder: recorder.clone(), + settle_ok, + recv_error, + }; + let outcome = block_on(run_source(svc, source, options)); + RunResult { + outcome, + events: recorder.events(), + } + } + + // --- tests -------------------------------------------------------------- + #[test] + fn success_dispatches_then_acks_in_order() { + let result = run(vec![event_message("ok", None)], RunOptions::idempotent()); + assert!(result.outcome.is_ok()); + // Handler effect is recorded before the ack: ack happens after success. + assert_eq!( + result.events, + vec![Event::Handled("ok".to_string()), Event::Ack] + ); + } + + #[test] + fn processes_every_message_then_stops_on_none() { + let result = run( + vec![event_message("ok", None), event_message("ok", None)], + RunOptions::idempotent(), + ); + assert!(result.outcome.is_ok()); + assert_eq!( + result.events, + vec![ + Event::Handled("ok".to_string()), + Event::Ack, + Event::Handled("ok".to_string()), + Event::Ack, + ] + ); + } + + #[test] + fn retryable_failure_nacks_without_acking() { + let result = run( + vec![event_message("retryable", None)], + RunOptions::idempotent(), + ); + assert!(result.outcome.is_ok()); + assert_eq!( + result.events.first(), + Some(&Event::Handled("retryable".to_string())) + ); + assert!(matches!(result.events.get(1), Some(Event::Nack(_)))); + assert!(!result.events.contains(&Event::Ack)); + } + + #[test] + fn permanent_failure_dead_letters_under_default_policy() { + let result = run( + vec![event_message("permanent", None)], + RunOptions::idempotent(), + ); + assert!(result.outcome.is_ok()); + assert_eq!( + result.events.first(), + Some(&Event::Handled("permanent".to_string())) + ); + assert!(matches!(result.events.get(1), Some(Event::DeadLetter(_)))); + } + + #[test] + fn permanent_failure_parks_under_park_policy() { + let result = run( + vec![event_message("permanent", None)], + RunOptions::idempotent().with_failure_policy(FailurePolicy::Park), + ); + assert!(result.outcome.is_ok()); + assert!(matches!(result.events.get(1), Some(Event::Park(_)))); + } + + #[test] + fn permanent_failure_logs_and_acks_under_log_and_ack_policy() { + let result = run( + vec![event_message("permanent", None)], + RunOptions::idempotent().with_failure_policy(FailurePolicy::LogAndAck), + ); + assert!(result.outcome.is_ok()); + assert_eq!(result.events.get(1), Some(&Event::Ack)); + } + + #[test] + fn permanent_failure_nacks_under_retry_policy() { + let result = run( + vec![event_message("permanent", None)], + RunOptions::idempotent().with_failure_policy(FailurePolicy::Retry), + ); + assert!(result.outcome.is_ok()); + assert!(matches!(result.events.get(1), Some(Event::Nack(_)))); + } + + #[test] + fn stop_policy_returns_error_without_settling() { + let result = run( + vec![event_message("permanent", None), event_message("ok", None)], + RunOptions::idempotent().with_failure_policy(FailurePolicy::Stop), + ); + let err = result + .outcome + .expect_err("stop policy should surface the error"); + assert!(err.is_permanent()); + // The handler ran, but the message was not settled and the second + // message was never processed. + assert_eq!(result.events, vec![Event::Handled("permanent".to_string())]); + } + + #[test] + fn inbox_mode_rejects_message_without_stable_id_before_dispatch() { + let result = run(vec![event_message("ok", None)], RunOptions::inbox(())); + assert!(result.outcome.is_ok()); + // Handler never ran (no Handled event); the missing id is a permanent + // failure routed to the default dead-letter policy, carrying the reason. + assert_eq!(result.events.len(), 1); + match &result.events[0] { + Event::DeadLetter(reason) => { + assert!(reason.contains("stable message id is required but missing")) + } + other => panic!("expected dead-letter, got {other:?}"), + } + assert!(!result.events.iter().any(|e| matches!(e, Event::Handled(_)))); + } + + #[test] + fn inbox_mode_dispatches_when_stable_id_is_present() { + let result = run( + vec![event_message("ok", Some("evt-1"))], + RunOptions::inbox(()), + ); + assert!(result.outcome.is_ok()); + assert_eq!( + result.events, + vec![Event::Handled("ok".to_string()), Event::Ack] + ); + } + + #[test] + fn recv_error_propagates_and_is_not_swallowed() { + let result = run_with( + vec![event_message("ok", None)], + RunOptions::idempotent(), + true, + true, + ); + let err = result.outcome.expect_err("recv error should propagate"); + assert!(err.is_retryable()); + assert!(result.events.is_empty()); + } + + #[test] + fn settle_error_propagates_and_is_not_swallowed() { + let result = run_with( + vec![event_message("ok", None)], + RunOptions::idempotent(), + false, + false, + ); + let err = result.outcome.expect_err("settle error should propagate"); + assert!(err.is_retryable()); + // The ack was attempted (recorded) before the error surfaced. + assert_eq!( + result.events, + vec![Event::Handled("ok".to_string()), Event::Ack] + ); + } + + #[test] + fn settle_error_on_failure_path_propagates() { + // A settle failure on the nack/failure-routing branch must propagate too, + // not just on the ack branch. + let result = run_with( + vec![event_message("retryable", None)], + RunOptions::idempotent(), + false, + false, + ); + let err = result + .outcome + .expect_err("nack settle error should propagate"); + assert!(err.is_retryable()); + assert_eq!( + result.events.first(), + Some(&Event::Handled("retryable".to_string())) + ); + assert!(matches!(result.events.get(1), Some(Event::Nack(_)))); + } + + #[test] + fn unhandled_message_is_acked_and_ignored() { + // No handler registered for "unrelated": ack-and-ignore, do not dispatch + // or dead-letter. + let result = run( + vec![event_message("unrelated", None), event_message("ok", None)], + RunOptions::idempotent(), + ); + assert!(result.outcome.is_ok()); + assert_eq!( + result.events, + vec![Event::Ack, Event::Handled("ok".to_string()), Event::Ack] + ); + } + + #[test] + fn run_source_future_is_send() { + // Guards the documented multi-threaded-executor contract for the common + // (no-inbox) path: the runner future must be Send. + fn assert_send(_: &T) {} + let recorder = Recorder::new(); + let svc = service(&recorder); + let source = FakeSource { + queue: VecDeque::new(), + recorder, + settle_ok: true, + recv_error: false, + }; + let future = run_source(svc, source, RunOptions::idempotent()); + assert_send(&future); + // Drive it to completion (empty source -> immediate Ok). + assert!(block_on(future).is_ok()); + } + + // A fake that relies on the trait's DEFAULT dead_letter/park (which forward + // to nack), proving the "never silently dropped" degrade-to-redelivery + // property of the provided methods. + struct DefaultReceived { + message: Message, + recorder: Arc, + } + + impl ReceivedMessage for DefaultReceived { + fn message(&self) -> &Message { + &self.message + } + async fn ack(self) -> Result<(), TransportError> { + self.recorder.push(Event::Ack); + Ok(()) + } + async fn nack(self, reason: &str) -> Result<(), TransportError> { + self.recorder.push(Event::Nack(reason.to_string())); + Ok(()) + } + // dead_letter and park intentionally NOT overridden. + } + + #[test] + fn default_dead_letter_and_park_degrade_to_nack() { + let recorder = Recorder::new(); + let dl = DefaultReceived { + message: event_message("ok", None), + recorder: recorder.clone(), + }; + block_on(dl.dead_letter("boom")).unwrap(); + + let park = DefaultReceived { + message: event_message("ok", None), + recorder: recorder.clone(), + }; + block_on(park.park("hold")).unwrap(); + + // Both defaults route to nack rather than dropping the message. + assert_eq!( + recorder.events(), + vec![ + Event::Nack("boom".to_string()), + Event::Nack("hold".to_string()) + ] + ); + } +} diff --git a/src/microsvc/transport/source.rs b/src/microsvc/transport/source.rs new file mode 100644 index 000000000..d1f6492df --- /dev/null +++ b/src/microsvc/transport/source.rs @@ -0,0 +1,82 @@ +//! Direct-transport receive traits. +//! +//! A direct broker client (Postgres, RabbitMQ, Kafka, NATS, or the in-memory +//! dev/test adapter) pulls messages with [`AsyncMessageSource`] and settles each +//! one through [`ReceivedMessage`]. The [`run_source`](super::run_source) runner +//! drives that loop: it dispatches through `Service::dispatch_message` and only +//! then asks the adapter to acknowledge. +//! +//! These are the *direct* receive shape. Knative / HTTP CloudEvents is a +//! separate ingress shape (the platform invokes an endpoint; there is no local +//! poll loop) and is intentionally not modeled through this trait. + +use std::future::Future; + +use super::{Message, TransportError}; + +/// A transport a runner can pull messages from, one at a time. +/// +/// `recv` resolves to: +/// - `Ok(Some(received))` — a message to dispatch and then settle; +/// - `Ok(None)` — the source is drained/closed; the runner stops **gracefully** +/// (this is the shutdown signal — an adapter wires its own stop into `recv`); +/// - `Err(e)` — a transport-level receive failure, surfaced by the runner rather +/// than swallowed. +/// +/// The future is `Send` so the runner can be driven on multi-threaded executors. +pub trait AsyncMessageSource: Send { + /// The settle handle for a received message. + type Received: ReceivedMessage; + + /// Receive the next message, if any. + fn recv( + &mut self, + ) -> impl Future, TransportError>> + Send + '_; +} + +/// A message received from a transport, plus the means to settle it. +/// +/// Settlement consumes the value so a message can be settled exactly once. `ack` +/// and `nack` are the universal primitives every transport supports; an adapter +/// maps them to its native operation (a row completion, a delivery ack, an +/// offset commit, a stream ack, …). `dead_letter` and `park` default to `nack` +/// so a message is never silently dropped; adapters with native dead-letter or +/// parking support should override them. +pub trait ReceivedMessage: Send { + /// The canonical message to dispatch. + fn message(&self) -> &Message; + + /// Acknowledge successful handling. The transport removes the message. + /// + /// The runner calls this only after consumer execution has succeeded (and, + /// in inbox mode, after the inbox receipt has committed). + fn ack(self) -> impl Future> + Send; + + /// Negatively acknowledge so the transport redelivers the message later. + fn nack(self, reason: &str) -> impl Future> + Send; + + /// Route the message to a dead-letter destination. + /// + /// Defaults to [`nack`](ReceivedMessage::nack): an adapter without a native + /// dead-letter destination keeps the message redeliverable rather than + /// dropping it, so a `DeadLetter` policy degrades to redelivery until the + /// adapter implements real dead-lettering. + fn dead_letter(self, reason: &str) -> impl Future> + Send + where + Self: Sized, + { + self.nack(reason) + } + + /// Hold the message for manual intervention without acking or redelivering. + /// + /// Defaults to [`nack`](ReceivedMessage::nack) for adapters without native + /// parking; such adapters keep the message redeliverable rather than + /// dropping it. + fn park(self, reason: &str) -> impl Future> + Send + where + Self: Sized, + { + self.nack(reason) + } +} diff --git a/src/microsvc/transport/stable_id.rs b/src/microsvc/transport/stable_id.rs new file mode 100644 index 000000000..e5a7c4b42 --- /dev/null +++ b/src/microsvc/transport/stable_id.rs @@ -0,0 +1,141 @@ +//! Stable message id validation for inbox-enabled execution. +//! +//! Idempotent consumers tolerate a missing [`Message.id`](super::super::Message) +//! because applying the same projection twice yields the same state. A consumer +//! *inbox*, however, deduplicates by a durable key, so inbox-enabled runs must +//! reject messages whose id cannot serve as that key. +//! +//! These rules define what counts as a usable stable id. The runner and the +//! future inbox implementation share them so "no stable id" is rejected +//! consistently rather than producing a silently broken dedup key. + +use std::error::Error; +use std::fmt; + +/// Maximum length, in bytes, of a stable message id. +/// +/// Bounds the size of dedup keys an inbox must store and index. CloudEvents +/// `id`, Kafka event-id headers, and outbox `message_id` values all fit well +/// under this ceiling. +pub const MAX_STABLE_MESSAGE_ID_LEN: usize = 512; + +/// Why a candidate stable message id is not usable for inbox deduplication. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum StableMessageIdError { + /// The message had no id at all (`None`). + Missing, + /// The id was present but empty or only whitespace. + Empty, + /// The id exceeded [`MAX_STABLE_MESSAGE_ID_LEN`]. + TooLong { + /// The actual length, in bytes. + len: usize, + }, + /// The id contained a control character (newline, NUL, etc.), which would + /// corrupt keys, logs, or headers. + InvalidCharacter, +} + +impl fmt::Display for StableMessageIdError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StableMessageIdError::Missing => { + write!(f, "stable message id is required but missing") + } + StableMessageIdError::Empty => { + write!(f, "stable message id is empty") + } + StableMessageIdError::TooLong { len } => write!( + f, + "stable message id is {len} bytes, exceeding the maximum of {MAX_STABLE_MESSAGE_ID_LEN}" + ), + StableMessageIdError::InvalidCharacter => { + write!(f, "stable message id contains a control character") + } + } + } +} + +impl Error for StableMessageIdError {} + +/// Validate a candidate stable message id for inbox-enabled execution. +/// +/// Returns the borrowed id when it is usable as a deduplication key. A missing, +/// empty, over-long, or control-character-bearing id is rejected. Surrounding +/// whitespace is checked for emptiness but the original value is returned +/// unchanged so callers key on exactly what the transport delivered. +pub fn validate_stable_message_id(id: Option<&str>) -> Result<&str, StableMessageIdError> { + let id = id.ok_or(StableMessageIdError::Missing)?; + if id.trim().is_empty() { + return Err(StableMessageIdError::Empty); + } + if id.len() > MAX_STABLE_MESSAGE_ID_LEN { + return Err(StableMessageIdError::TooLong { len: id.len() }); + } + if id.chars().any(|c| c.is_control()) { + return Err(StableMessageIdError::InvalidCharacter); + } + Ok(id) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_a_well_formed_id() { + assert_eq!(validate_stable_message_id(Some("evt-1")), Ok("evt-1")); + // Surrounding whitespace is preserved in the returned value. + assert_eq!(validate_stable_message_id(Some(" evt-1 ")), Ok(" evt-1 ")); + } + + #[test] + fn rejects_missing_id() { + assert_eq!( + validate_stable_message_id(None), + Err(StableMessageIdError::Missing) + ); + } + + #[test] + fn rejects_empty_or_whitespace_id() { + assert_eq!( + validate_stable_message_id(Some("")), + Err(StableMessageIdError::Empty) + ); + assert_eq!( + validate_stable_message_id(Some(" ")), + Err(StableMessageIdError::Empty) + ); + } + + #[test] + fn rejects_over_long_id() { + let id = "a".repeat(MAX_STABLE_MESSAGE_ID_LEN + 1); + assert_eq!( + validate_stable_message_id(Some(&id)), + Err(StableMessageIdError::TooLong { + len: MAX_STABLE_MESSAGE_ID_LEN + 1 + }) + ); + // The boundary length is accepted. + let boundary = "a".repeat(MAX_STABLE_MESSAGE_ID_LEN); + assert!(validate_stable_message_id(Some(&boundary)).is_ok()); + } + + #[test] + fn rejects_control_characters() { + assert_eq!( + validate_stable_message_id(Some("evt\n1")), + Err(StableMessageIdError::InvalidCharacter) + ); + assert_eq!( + validate_stable_message_id(Some("evt\u{0}1")), + Err(StableMessageIdError::InvalidCharacter) + ); + assert_eq!( + validate_stable_message_id(Some("evt\t1")), + Err(StableMessageIdError::InvalidCharacter) + ); + } +} diff --git a/src/outbox_worker/store.rs b/src/outbox_worker/store.rs index 2a8c77047..f15cc6ac7 100644 --- a/src/outbox_worker/store.rs +++ b/src/outbox_worker/store.rs @@ -22,6 +22,11 @@ pub struct ClaimOutboxMessages { pub batch_size: usize, pub lease: Duration, pub destination: Option, + /// Restrict the claim to this explicit set of message ids. `None` claims the + /// next claimable batch in created-at order (normal worker polling); `Some` + /// claims only the listed ids (after-commit immediate dispatch). Ids that are + /// not currently claimable are simply skipped — a raced id is not an error. + pub message_ids: Option>, } impl ClaimOutboxMessages { @@ -31,6 +36,7 @@ impl ClaimOutboxMessages { batch_size, lease, destination: None, + message_ids: None, } } @@ -38,6 +44,26 @@ impl ClaimOutboxMessages { self.destination = Some(destination.into()); self } + + /// Restrict this claim to an explicit list of message ids (after-commit + /// immediate dispatch). The batch size is bounded by the id count. + pub fn for_ids(worker_id: impl Into, ids: Vec, lease: Duration) -> Self { + Self { + worker_id: worker_id.into(), + batch_size: ids.len(), + lease, + destination: None, + message_ids: Some(ids), + } + } + + /// Whether `id` is selectable under this request's id filter. + fn selects(&self, id: &str) -> bool { + match &self.message_ids { + Some(ids) => ids.iter().any(|wanted| wanted == id), + None => true, + } + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -269,6 +295,10 @@ impl OutboxStore for HashMapOutboxStore { let ids = claim_order_ids(storage.values()); let mut claimed = Vec::new(); for id in ids { + if !request.selects(&id) { + continue; + } + let Some(message) = storage.get_mut(&id) else { continue; }; @@ -372,6 +402,10 @@ impl AsyncOutboxStore for HashMapOutboxStore { let mut claimed = Vec::new(); for id in ids { + if !request.selects(&id) { + continue; + } + let Some(message) = storage.get_mut(&id) else { continue; }; @@ -590,6 +624,64 @@ mod tests { assert_eq!(claimed[0].id(), "msg-z"); } + #[test] + fn claim_by_explicit_ids_claims_only_requested() { + let repo = HashMapRepository::new(); + store_message( + &repo, + OutboxMessage::create("msg-a", "Event", b"{}".to_vec()).unwrap(), + ); + store_message( + &repo, + OutboxMessage::create("msg-b", "Event", b"{}".to_vec()).unwrap(), + ); + store_message( + &repo, + OutboxMessage::create("msg-c", "Event", b"{}".to_vec()).unwrap(), + ); + + let claimed = repo + .outbox_store() + .claim(ClaimOutboxMessages::for_ids( + "worker-1", + vec!["msg-b".to_string(), "msg-c".to_string()], + Duration::from_secs(60), + )) + .unwrap(); + + let mut claimed_ids = claimed + .iter() + .map(|m| m.id().to_string()) + .collect::>(); + claimed_ids.sort(); + assert_eq!(claimed_ids, vec!["msg-b".to_string(), "msg-c".to_string()]); + // The unrequested row stays pending. + assert!(load_message(&repo, "msg-a").is_pending()); + } + + #[test] + fn claim_by_ids_skips_unclaimable_without_error() { + let repo = HashMapRepository::new(); + let mut leased = OutboxMessage::create("msg-a", "Event", b"{}".to_vec()).unwrap(); + leased + .claim_for("other-worker", Duration::from_secs(60)) + .unwrap(); + store_message(&repo, leased); + + // Requesting a currently-leased id (and a missing id) yields no claim, + // not an error. + let claimed = repo + .outbox_store() + .claim(ClaimOutboxMessages::for_ids( + "worker-1", + vec!["msg-a".to_string(), "missing".to_string()], + Duration::from_secs(60), + )) + .unwrap(); + + assert!(claimed.is_empty()); + } + #[test] fn sort_by_claim_order_uses_message_id_tiebreaker() { let mut later = OutboxMessage::create("msg-c", "Event", b"{}".to_vec()).unwrap(); diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 3ae670e83..4d26359f8 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -453,6 +453,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { OR (status = $3 AND (claimed_until IS NULL OR claimed_until <= to_timestamp($2))) ) AND ($4::text IS NULL OR destination = $4) + AND ($9::text[] IS NULL OR message_id = ANY($9::text[])) ORDER BY created_at ASC, message_id ASC LIMIT $5 FOR UPDATE SKIP LOCKED @@ -493,6 +494,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { .bind(OutboxMessageStatus::InFlight.as_str()) .bind(&request.worker_id) .bind(claimed_until_epoch) + .bind(request.message_ids.as_deref()) .fetch_all(&mut *tx) .await .map_err(|err| repository_storage_error("claim outbox messages", err))?; diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index b62dafa25..7c09dc5e2 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -432,41 +432,57 @@ impl AsyncOutboxStore for SqliteOutboxStore { repository_storage_error("begin outbox claim transaction", err) })?; - let limit = sqlx_repository_i64_from_u64( - SQLITE_BACKEND, - request.batch_size as u64, - "outbox claim limit", - SIGNED_INTEGER_STORAGE, - )?; - let candidate_rows = sqlx::query( - r#" - SELECT message_id - FROM outbox_messages - WHERE ( - (status = ? AND CAST(next_available_at AS REAL) <= ?) - OR (status = ? AND (claimed_until IS NULL OR CAST(claimed_until AS REAL) <= ?)) + // Explicit ids (after-commit immediate dispatch) bypass the ordered + // candidate scan; the per-id conditional UPDATE below still enforces + // claimability and destination, so raced/unclaimable ids are skipped. + let candidate_ids: Vec = if let Some(ids) = request.message_ids.clone() { + ids + } else { + let limit = sqlx_repository_i64_from_u64( + SQLITE_BACKEND, + request.batch_size as u64, + "outbox claim limit", + SIGNED_INTEGER_STORAGE, + )?; + let candidate_rows = sqlx::query( + r#" + SELECT message_id + FROM outbox_messages + WHERE ( + (status = ? AND CAST(next_available_at AS REAL) <= ?) + OR (status = ? AND (claimed_until IS NULL OR CAST(claimed_until AS REAL) <= ?)) + ) + AND (? IS NULL OR destination = ?) + ORDER BY CAST(created_at AS REAL) ASC, message_id ASC + LIMIT ? + "#, ) - AND (? IS NULL OR destination = ?) - ORDER BY CAST(created_at AS REAL) ASC, message_id ASC - LIMIT ? - "#, - ) - .bind(OutboxMessageStatus::Pending.as_str()) - .bind(now_epoch) - .bind(OutboxMessageStatus::InFlight.as_str()) - .bind(now_epoch) - .bind(request.destination.as_deref()) - .bind(request.destination.as_deref()) - .bind(limit) - .fetch_all(&mut *tx) - .await - .map_err(|err| repository_storage_error("select claimable outbox messages", err))?; + .bind(OutboxMessageStatus::Pending.as_str()) + .bind(now_epoch) + .bind(OutboxMessageStatus::InFlight.as_str()) + .bind(now_epoch) + .bind(request.destination.as_deref()) + .bind(request.destination.as_deref()) + .bind(limit) + .fetch_all(&mut *tx) + .await + .map_err(|err| { + repository_storage_error("select claimable outbox messages", err) + })?; + let mut ids = Vec::with_capacity(candidate_rows.len()); + for row in candidate_rows { + ids.push(row.try_get::("message_id").map_err(|err| { + repository_storage_error("decode outbox message id row", err) + })?); + } + ids + }; let mut claimed = Vec::new(); - for row in candidate_rows { - let message_id: String = row - .try_get("message_id") - .map_err(|err| repository_storage_error("decode outbox message id row", err))?; + for message_id in candidate_ids { + if claimed.len() >= request.batch_size { + break; + } let result = sqlx::query( r#" UPDATE outbox_messages diff --git a/tests/hashmap_repository_conformance/main.rs b/tests/hashmap_repository_conformance/main.rs index 1d820db12..1714e1586 100644 --- a/tests/hashmap_repository_conformance/main.rs +++ b/tests/hashmap_repository_conformance/main.rs @@ -89,3 +89,13 @@ async fn worker_claim_complete_and_retry_lifecycle() { ) .await; } + +#[tokio::test] +async fn worker_claim_by_ids_claims_only_requested() { + let repo = repository(); + conformance::outbox::worker_claim_by_ids_claims_only_requested( + repo.clone(), + repo.outbox_store(), + ) + .await; +} diff --git a/tests/persistent_repository_conformance/outbox.rs b/tests/persistent_repository_conformance/outbox.rs index 91b2887d7..9077c2876 100644 --- a/tests/persistent_repository_conformance/outbox.rs +++ b/tests/persistent_repository_conformance/outbox.rs @@ -262,6 +262,77 @@ where assert_eq!(failed.last_error.as_deref(), Some("second failure")); } +pub async fn worker_claim_by_ids_claims_only_requested(repo: R, outbox: S) +where + R: AsyncGetStream + AsyncTransactionalCommit + Clone + Send + Sync + 'static, + S: AsyncOutboxStore + Send + Sync, +{ + let wanted_id = unique_id("wanted-outbox"); + let other_id = unique_id("other-outbox"); + for (message_id, seat_id) in [(&wanted_id, "wanted-seat"), (&other_id, "other-seat")] { + let mut seat = added_seat(&unique_id(seat_id)); + let message = OutboxMessage::create(message_id, "SeatAdded", b"{}".to_vec()) + .expect("outbox message should be valid"); + repo.clone() + .async_aggregate::() + .outbox(message) + .commit(&mut seat) + .await + .expect("message should be stored"); + } + + // Claiming by explicit id claims only the requested row (claim order among + // an explicit id set is unspecified across backends). + let claimed = outbox + .claim_async(ClaimOutboxMessages::for_ids( + "immediate-worker", + vec![wanted_id.clone()], + Duration::from_secs(60), + )) + .await + .expect("claim by id should succeed"); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].id(), wanted_id); + + // The unrequested row remains claimable by a normal poll. + let other = find_outbox_by_id(&outbox, &other_id) + .await + .expect("other message should exist"); + assert_eq!(other.status, OutboxMessageStatus::Pending); + + // A raced/missing id yields an empty claim, not an error. + let empty = outbox + .claim_async(ClaimOutboxMessages::for_ids( + "immediate-worker", + vec![unique_id("never-stored")], + Duration::from_secs(60), + )) + .await + .expect("claiming a missing id should not error"); + assert!(empty.is_empty()); + + // A requested id that is leased by another worker must be skipped, not + // stolen: this is the claim-safety property of the by-id path (it exercises + // the SQLite per-id conditional UPDATE and the Postgres claimability CTE). + let leased = outbox + .claim_async(ClaimOutboxMessages::for_ids( + "worker-b", + vec![wanted_id.clone()], + Duration::from_secs(60), + )) + .await + .expect("claiming a live-leased id should not error"); + assert!( + leased.is_empty(), + "by-id claim must not steal a row already leased by another worker" + ); + let still_owned = find_outbox_by_id(&outbox, &wanted_id) + .await + .expect("leased message should exist"); + assert_eq!(still_owned.status, OutboxMessageStatus::InFlight); + assert_eq!(still_owned.worker_id.as_deref(), Some("immediate-worker")); +} + fn added_seat(id: &str) -> Seat { let mut seat = Seat::default(); seat.add(id.to_string(), "floor".to_string()) diff --git a/tests/postgres_repository_conformance/main.rs b/tests/postgres_repository_conformance/main.rs index 04d2877bb..688016a3a 100644 --- a/tests/postgres_repository_conformance/main.rs +++ b/tests/postgres_repository_conformance/main.rs @@ -112,3 +112,15 @@ async fn worker_claim_complete_and_retry_lifecycle() { ) .await; } + +#[tokio::test] +async fn worker_claim_by_ids_claims_only_requested() { + let Some(repo) = repository().await else { + return; + }; + conformance::outbox::worker_claim_by_ids_claims_only_requested( + repo.clone(), + repo.outbox_store(), + ) + .await; +} diff --git a/tests/sqlite_repository_conformance/main.rs b/tests/sqlite_repository_conformance/main.rs index e1c7356f3..e1b148c7a 100644 --- a/tests/sqlite_repository_conformance/main.rs +++ b/tests/sqlite_repository_conformance/main.rs @@ -82,3 +82,13 @@ async fn worker_claim_complete_and_retry_lifecycle() { ) .await; } + +#[tokio::test] +async fn worker_claim_by_ids_claims_only_requested() { + let repo = repository().await; + conformance::outbox::worker_claim_by_ids_claims_only_requested( + repo.clone(), + repo.outbox_store(), + ) + .await; +} From bb102710dec17516bd803f7118f35a7fa89860b3 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 21:20:06 -0500 Subject: [PATCH 03/39] test(transport): reusable conformance harness + in-memory reference run Adds tests/transport_conformance/mod.rs (adapter-neutral fakes: FakeSource, FakeReceived, FakePublisher, recording service, plus source-runner and outbox dispatcher contract fns) and the tests/transport_in_memory target that runs the full contract against the in-memory reference. Concrete adapters reuse the harness via #[path]. Adds OutboxDispatcher::publisher()/store() accessors. Implements [[tasks/async-transport-conformance-tests]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/outbox_dispatch.rs | 10 + tests/transport_conformance/mod.rs | 451 ++++++++++++++++++++++ tests/transport_in_memory/main.rs | 72 ++++ 3 files changed, 533 insertions(+) create mode 100644 tests/transport_conformance/mod.rs create mode 100644 tests/transport_in_memory/main.rs diff --git a/src/microsvc/transport/outbox_dispatch.rs b/src/microsvc/transport/outbox_dispatch.rs index c9c86b84f..05f685d70 100644 --- a/src/microsvc/transport/outbox_dispatch.rs +++ b/src/microsvc/transport/outbox_dispatch.rs @@ -164,6 +164,16 @@ where } } + /// The publisher this dispatcher sends through. + pub fn publisher(&self) -> &P { + &self.publisher + } + + /// The outbox store this dispatcher claims from. + pub fn store(&self) -> &S { + &self.store + } + /// Immediate after-commit dispatch of the explicit outbox ids a commit just /// inserted. Claims those ids (raced/unclaimable ids are skipped, not an /// error) before publishing, so it never races the polling worker. diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs new file mode 100644 index 000000000..7cd25357b --- /dev/null +++ b/tests/transport_conformance/mod.rs @@ -0,0 +1,451 @@ +//! Reusable async-transport conformance harness. +//! +//! Adapter-neutral fakes plus a contract suite that proves the shared transport +//! behaviour (source runner ack/nack/failure ordering, stable-id handling, and +//! the outbox publisher/dispatcher thresholds) before any real broker exists. +//! +//! Concrete adapters reuse the pieces here: a real *source* adapter can be +//! exercised against [`FakePublisher`], a real *publisher* adapter against +//! [`FakeSource`], and any [`AsyncOutboxStore`] against the dispatcher contract. +//! Other test targets include this module with +//! `#[path = "../transport_conformance/mod.rs"] mod conformance;`. +#![allow(dead_code)] + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; +use sourced_rust::microsvc::transport::{ + run_source, AsyncMessagePublisher, AsyncMessageSource, FailurePolicy, OutboxDispatcher, + ReceivedMessage, RunOptions, TransportError, +}; +use sourced_rust::microsvc::{HandlerError, Message, MessageKind, Service}; +use sourced_rust::{ + CommitBatch, HashMapOutboxStore, HashMapRepository, OutboxMessage, OutboxMessageStatus, + TransactionalCommit, +}; + +/// One observable transport effect, recorded in order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Event { + /// A handler ran for the named message. + Handled(String), + Ack, + Nack(String), + DeadLetter(String), + Park(String), +} + +/// Ordered recorder shared by the service handlers and the fake transport. +#[derive(Default)] +pub struct Recorder { + events: Mutex>, +} + +impl Recorder { + pub fn new() -> Arc { + Arc::new(Self::default()) + } + pub fn push(&self, event: Event) { + self.events.lock().unwrap().push(event); + } + pub fn events(&self) -> Vec { + self.events.lock().unwrap().clone() + } +} + +/// A received message that records how it was settled. +pub struct FakeReceived { + message: Message, + recorder: Arc, + settle_ok: bool, +} + +impl FakeReceived { + fn settle(self, event: Event) -> Result<(), TransportError> { + self.recorder.push(event); + if self.settle_ok { + Ok(()) + } else { + Err(TransportError::retryable("settle failed")) + } + } +} + +impl ReceivedMessage for FakeReceived { + fn message(&self) -> &Message { + &self.message + } + async fn ack(self) -> Result<(), TransportError> { + self.settle(Event::Ack) + } + async fn nack(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::Nack(reason.to_string())) + } + async fn dead_letter(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::DeadLetter(reason.to_string())) + } + async fn park(self, reason: &str) -> Result<(), TransportError> { + self.settle(Event::Park(reason.to_string())) + } +} + +/// A source that yields a preset queue of messages, then `None`. +pub struct FakeSource { + queue: VecDeque, + recorder: Arc, + settle_ok: bool, + recv_error: bool, +} + +impl FakeSource { + pub fn new(recorder: Arc, messages: Vec) -> Self { + Self { + queue: messages.into_iter().collect(), + recorder, + settle_ok: true, + recv_error: false, + } + } + pub fn with_settle_failure(mut self) -> Self { + self.settle_ok = false; + self + } + pub fn with_recv_error(mut self) -> Self { + self.recv_error = true; + self + } +} + +impl AsyncMessageSource for FakeSource { + type Received = FakeReceived; + async fn recv(&mut self) -> Result, TransportError> { + if self.recv_error { + return Err(TransportError::retryable("recv failed")); + } + Ok(self.queue.pop_front().map(|message| FakeReceived { + message, + recorder: self.recorder.clone(), + settle_ok: self.settle_ok, + })) + } +} + +/// How a [`FakePublisher`] resolves each publish. +#[derive(Debug, Clone, Copy)] +pub enum PublishMode { + Succeed, + /// Unknown outcome: a retryable error (the row must stay retryable). + FailUnknown, +} + +/// A publisher that records published message ids and follows a [`PublishMode`]. +pub struct FakePublisher { + published: Mutex>, + mode: PublishMode, +} + +impl FakePublisher { + pub fn new(mode: PublishMode) -> Self { + Self { + published: Mutex::new(Vec::new()), + mode, + } + } + pub fn published_ids(&self) -> Vec { + self.published.lock().unwrap().clone() + } +} + +impl AsyncMessagePublisher for FakePublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + match self.mode { + PublishMode::Succeed => { + self.published + .lock() + .unwrap() + .push(message.id().unwrap_or_default().to_string()); + Ok(()) + } + PublishMode::FailUnknown => Err(TransportError::retryable("publish outcome unknown")), + } + } +} + +/// A service with conventional handlers: `ok` succeeds, `retryable` fails with a +/// retryable error, `permanent` fails with a permanent error. Each records that +/// it ran so tests can assert dispatch-before-ack ordering. +pub fn recording_service(recorder: &Arc) -> Arc> { + let ok = recorder.clone(); + let retryable = recorder.clone(); + let permanent = recorder.clone(); + Arc::new( + Service::new(()) + .event("ok") + .handle(move |ctx| { + ok.push(Event::Handled(ctx.message().name().to_string())); + Ok(json!({})) + }) + .event("retryable") + .handle(move |ctx| { + retryable.push(Event::Handled(ctx.message().name().to_string())); + Err(HandlerError::Other("infra".into())) + }) + .event("permanent") + .handle(move |ctx| { + permanent.push(Event::Handled(ctx.message().name().to_string())); + Err(HandlerError::Rejected("nope".into())) + }), + ) +} + +pub fn event_message(name: &str, id: Option<&str>) -> Message { + let mut message = Message::new(name, MessageKind::Event, b"{}".to_vec()); + if let Some(id) = id { + message = message.with_id(id); + } + message +} + +// ============================================================================= +// Source-runner contract +// ============================================================================= + +pub async fn source_dispatches_before_ack() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new(recorder.clone(), vec![event_message("ok", Some("m1"))]); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + assert_eq!( + recorder.events(), + vec![Event::Handled("ok".into()), Event::Ack], + "handler must run before ack" + ); +} + +pub async fn source_retryable_failure_nacks_without_ack() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new( + recorder.clone(), + vec![event_message("retryable", Some("m1"))], + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + let events = recorder.events(); + assert_eq!(events.first(), Some(&Event::Handled("retryable".into()))); + assert!(matches!(events.get(1), Some(Event::Nack(_)))); + assert!(!events.contains(&Event::Ack)); +} + +pub async fn source_permanent_failure_dead_letters_by_default() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new( + recorder.clone(), + vec![event_message("permanent", Some("m1"))], + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + assert!(matches!( + recorder.events().get(1), + Some(Event::DeadLetter(_)) + )); +} + +pub async fn source_permanent_failure_stops_under_stop_policy() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new( + recorder.clone(), + vec![ + event_message("permanent", Some("m1")), + event_message("ok", Some("m2")), + ], + ); + let outcome = run_source( + service, + source, + RunOptions::idempotent().with_failure_policy(FailurePolicy::Stop), + ) + .await; + assert!(outcome.unwrap_err().is_permanent()); + // Second message never processed; first was not settled. + assert_eq!(recorder.events(), vec![Event::Handled("permanent".into())]); +} + +pub async fn source_unhandled_message_is_acked_and_ignored() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new( + recorder.clone(), + vec![event_message("unrelated", Some("m1"))], + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + // Acked without dispatching or dead-lettering. + assert_eq!(recorder.events(), vec![Event::Ack]); +} + +pub async fn source_inbox_mode_rejects_missing_stable_id() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + // No id on the message; inbox mode requires a stable id. + let source = FakeSource::new(recorder.clone(), vec![event_message("ok", None)]); + run_source(service, source, RunOptions::inbox(())) + .await + .unwrap(); + let events = recorder.events(); + // Handler never ran; the missing id is a permanent failure (dead-lettered). + assert!(!events.iter().any(|e| matches!(e, Event::Handled(_)))); + assert!(matches!(events.first(), Some(Event::DeadLetter(_)))); +} + +pub async fn source_inbox_mode_dispatches_with_stable_id() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new(recorder.clone(), vec![event_message("ok", Some("m1"))]); + run_source(service, source, RunOptions::inbox(())) + .await + .unwrap(); + assert_eq!( + recorder.events(), + vec![Event::Handled("ok".into()), Event::Ack] + ); +} + +pub async fn source_propagates_recv_errors() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = + FakeSource::new(recorder.clone(), vec![event_message("ok", Some("m1"))]).with_recv_error(); + let outcome = run_source(service, source, RunOptions::idempotent()).await; + assert!(outcome.is_err(), "recv errors must not be swallowed"); + assert!(recorder.events().is_empty()); +} + +pub async fn source_propagates_settle_errors() { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new(recorder.clone(), vec![event_message("ok", Some("m1"))]) + .with_settle_failure(); + let outcome = run_source(service, source, RunOptions::idempotent()).await; + assert!(outcome.is_err(), "settle errors must not be swallowed"); + // The ack was attempted before the error surfaced. + assert_eq!( + recorder.events(), + vec![Event::Handled("ok".into()), Event::Ack] + ); +} + +// ============================================================================= +// Publisher / outbox dispatcher contract +// ============================================================================= + +fn store_outbox(repo: &HashMapRepository, id: &str) -> String { + let message = OutboxMessage::create(id, "OrderCreated", b"\x01".to_vec()).unwrap(); + let mut batch = CommitBatch::empty(); + batch.outbox_messages.push(message); + repo.commit_batch(batch).unwrap(); + id.to_string() +} + +fn outbox_status(repo: &HashMapRepository, id: &str) -> Option { + use sourced_rust::OutboxStore; + let store = repo.outbox_store(); + [ + OutboxMessageStatus::Pending, + OutboxMessageStatus::InFlight, + OutboxMessageStatus::Published, + OutboxMessageStatus::Failed, + ] + .into_iter() + .find(|status| { + store + .messages_by_status(status.clone()) + .unwrap() + .iter() + .any(|message| message.id() == id) + }) +} + +fn dispatcher( + repo: &HashMapRepository, + mode: PublishMode, + max_attempts: u32, +) -> OutboxDispatcher { + OutboxDispatcher::new( + repo.outbox_store(), + FakePublisher::new(mode), + "immediate:conformance", + Duration::from_secs(60), + max_attempts, + ) +} + +pub async fn dispatcher_completes_only_after_publish_success() { + let repo = HashMapRepository::new(); + let id = store_outbox(&repo, "evt-1"); + let dispatcher = dispatcher(&repo, PublishMode::Succeed, 3); + + let outcome = dispatcher + .dispatch_ids(std::slice::from_ref(&id)) + .await + .unwrap(); + assert_eq!(outcome.published, 1); + assert_eq!( + dispatcher.publisher().published_ids(), + vec!["evt-1".to_string()] + ); + assert_eq!( + outbox_status(&repo, &id), + Some(OutboxMessageStatus::Published) + ); +} + +pub async fn dispatcher_unknown_outcome_stays_retryable() { + let repo = HashMapRepository::new(); + let id = store_outbox(&repo, "evt-1"); + let dispatcher = dispatcher(&repo, PublishMode::FailUnknown, 3); + + let outcome = dispatcher + .dispatch_ids(std::slice::from_ref(&id)) + .await + .unwrap(); + assert_eq!(outcome.published, 0); + assert_eq!(outcome.released, 1); + assert_eq!( + outbox_status(&repo, &id), + Some(OutboxMessageStatus::Pending), + "row must stay retryable" + ); +} + +pub async fn dispatcher_claims_explicit_ids_before_publish() { + let repo = HashMapRepository::new(); + let wanted = store_outbox(&repo, "evt-1"); + let other = store_outbox(&repo, "evt-2"); + let dispatcher = dispatcher(&repo, PublishMode::Succeed, 3); + + let outcome = dispatcher + .dispatch_ids(std::slice::from_ref(&wanted)) + .await + .unwrap(); + assert_eq!(outcome.claimed, 1); + assert_eq!(outcome.published, 1); + assert_eq!( + outbox_status(&repo, &wanted), + Some(OutboxMessageStatus::Published) + ); + // The unrequested row is untouched (claimed before publish, by id). + assert_eq!( + outbox_status(&repo, &other), + Some(OutboxMessageStatus::Pending) + ); +} diff --git a/tests/transport_in_memory/main.rs b/tests/transport_in_memory/main.rs new file mode 100644 index 000000000..6fa369a2f --- /dev/null +++ b/tests/transport_in_memory/main.rs @@ -0,0 +1,72 @@ +//! In-memory reference run of the async-transport conformance harness. +//! +//! Proves the shared transport contract (source runner + outbox dispatcher) +//! against adapter-neutral fakes, with no external broker. Concrete adapters +//! reuse the harness in [`transport_conformance`] for their own targets. + +#[path = "../transport_conformance/mod.rs"] +mod conformance; + +// --- source runner contract ------------------------------------------------- + +#[tokio::test] +async fn source_dispatches_before_ack() { + conformance::source_dispatches_before_ack().await; +} + +#[tokio::test] +async fn source_retryable_failure_nacks_without_ack() { + conformance::source_retryable_failure_nacks_without_ack().await; +} + +#[tokio::test] +async fn source_permanent_failure_dead_letters_by_default() { + conformance::source_permanent_failure_dead_letters_by_default().await; +} + +#[tokio::test] +async fn source_permanent_failure_stops_under_stop_policy() { + conformance::source_permanent_failure_stops_under_stop_policy().await; +} + +#[tokio::test] +async fn source_unhandled_message_is_acked_and_ignored() { + conformance::source_unhandled_message_is_acked_and_ignored().await; +} + +#[tokio::test] +async fn source_inbox_mode_rejects_missing_stable_id() { + conformance::source_inbox_mode_rejects_missing_stable_id().await; +} + +#[tokio::test] +async fn source_inbox_mode_dispatches_with_stable_id() { + conformance::source_inbox_mode_dispatches_with_stable_id().await; +} + +#[tokio::test] +async fn source_propagates_recv_errors() { + conformance::source_propagates_recv_errors().await; +} + +#[tokio::test] +async fn source_propagates_settle_errors() { + conformance::source_propagates_settle_errors().await; +} + +// --- publisher / outbox dispatcher contract --------------------------------- + +#[tokio::test] +async fn dispatcher_completes_only_after_publish_success() { + conformance::dispatcher_completes_only_after_publish_success().await; +} + +#[tokio::test] +async fn dispatcher_unknown_outcome_stays_retryable() { + conformance::dispatcher_unknown_outcome_stays_retryable().await; +} + +#[tokio::test] +async fn dispatcher_claims_explicit_ids_before_publish() { + conformance::dispatcher_claims_explicit_ids_before_publish().await; +} From 70298e103b4209f79bf4fef7362e0dc9dfc7b92a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 21:52:48 -0500 Subject: [PATCH 04/39] feat(transport): Postgres durable receive via OutboxSource OutboxSource turns any outbox store into an AsyncMessageSource (claim -> Message -> settle by row status: ack=complete, nack=release-for-retry, dead-letter/park=fail). OutboxSource is the Postgres starter transport (outbox-backed mode, FOR UPDATE SKIP LOCKED + lease, no new table). Adds tests/postgres_transport integration tests (verified against real Postgres: drain, concurrent SKIP-LOCKED claim safety, retry, dead-letter), wires postgres_transport into the Postgres CI job, and adds RabbitMQ/Kafka/ NATS services to compose.yaml for local integration testing. sqlxmq was evaluated (owner suggestion) and not adopted: its push-based JobRegistry conflicts with our pull-based AsyncMessageSource/run_source boundary; patterns borrowed per the spec, not the crate. Implements [[tasks/postgres-transport-adapter-first-pass]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/on-pr-quality.yaml | 1 + compose.yaml | 61 +++++ src/microsvc/transport/mod.rs | 4 + src/microsvc/transport/outbox_source.rs | 305 ++++++++++++++++++++++++ tests/postgres_transport/main.rs | 185 ++++++++++++++ 5 files changed, 556 insertions(+) create mode 100644 src/microsvc/transport/outbox_source.rs create mode 100644 tests/postgres_transport/main.rs diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index d901547de..7534ad827 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -42,4 +42,5 @@ jobs: run: | cargo test --test postgres_repository --all-features --verbose cargo test --test postgres_repository_conformance --all-features --verbose + cargo test --test postgres_transport --all-features --verbose cargo test --test distributed_read_model --all-features --verbose diff --git a/compose.yaml b/compose.yaml index 0780cb5cc..46ed95635 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,19 @@ +# Local dependencies for development and transport integration tests. +# +# docker compose up -d # start all +# docker compose up -d postgres # just one +# +# Then point each test's env var at the service and run the feature-gated target +# (each integration test skips when its env var is unset): +# +# DATABASE_URL=postgres://sourced:sourced@localhost:5432/sourced_rust \ +# cargo test --test postgres_transport --features postgres +# AMQP_URL=amqp://guest:guest@localhost:5672/%2f \ +# cargo test --test rabbitmq_transport --features rabbitmq +# KAFKA_BROKERS=localhost:9092 \ +# cargo test --test kafka_transport --features kafka +# NATS_URL=nats://localhost:4222 \ +# cargo test --test nats_transport --features nats services: postgres: image: postgres:16 @@ -12,3 +28,48 @@ services: interval: 2s timeout: 5s retries: 20 + + rabbitmq: + image: rabbitmq:3.13-management-alpine + ports: + - "5672:5672" # AMQP + - "15672:15672" # management UI + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + + kafka: + image: apache/kafka:3.8.0 + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + healthcheck: + test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 || exit 1"] + interval: 10s + timeout: 10s + retries: 30 + + nats: + image: nats:2.10-alpine + command: ["-js", "-m", "8222"] # JetStream + HTTP monitoring (for healthcheck) + ports: + - "4222:4222" + - "8222:8222" + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://localhost:8222/healthz || exit 1"] + interval: 5s + timeout: 5s + retries: 20 diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index fa299908e..7548a7a47 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -82,6 +82,7 @@ mod capabilities; mod error; mod failure_policy; mod outbox_dispatch; +mod outbox_source; mod publisher; mod run_options; mod runner; @@ -92,6 +93,9 @@ pub use capabilities::{ConsumerAckKind, KnativeIntegrationKind, TransportCapabil pub use error::{TransportError, TransportErrorKind}; pub use failure_policy::{FailureAction, FailurePolicy}; pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; +pub use outbox_source::{ + OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, +}; pub use publisher::AsyncMessagePublisher; pub use run_options::{ConsumerDeliveryMode, InboxHook, NoInbox, RunOptions}; pub use runner::run_source; diff --git a/src/microsvc/transport/outbox_source.rs b/src/microsvc/transport/outbox_source.rs new file mode 100644 index 000000000..22b33f54d --- /dev/null +++ b/src/microsvc/transport/outbox_source.rs @@ -0,0 +1,305 @@ +//! Outbox-backed durable receive. +//! +//! [`OutboxSource`] turns any [`AsyncOutboxStore`] into an [`AsyncMessageSource`]: +//! it claims durable rows (`FOR UPDATE SKIP LOCKED` + lease in the SQL stores), +//! maps each to a canonical [`Message`], and settles by row status — +//! ack→complete, nack→release-for-retry, dead-letter/park→fail (the terminal +//! DLQ/archive state). `OutboxSource` is the Postgres +//! "starter" durable transport; the same type works over the in-memory and +//! SQLite stores for tests. +//! +//! `recv` drains the currently-claimable rows and then returns `Ok(None)`, so +//! `run_source` processes a finite backlog and stops. A long-running consumer +//! wraps `run_source` in a poll loop (waking on `LISTEN`/`NOTIFY` where the store +//! supports it, with polling authoritative); that daemon is a thin runtime +//! wrapper layered on top of this pure, runtime-agnostic source. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::TransportError; +use crate::microsvc::Message; +use crate::outbox::OutboxMessage; +use crate::outbox_worker::{AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef}; + +/// Default lease held on a claimed row while it is being dispatched. +pub const DEFAULT_OUTBOX_SOURCE_LEASE: Duration = Duration::from_secs(30); +/// Default number of rows claimed per `recv` refill. +pub const DEFAULT_OUTBOX_SOURCE_BATCH: usize = 16; + +/// An [`AsyncMessageSource`] backed by an [`AsyncOutboxStore`]. +pub struct OutboxSource { + store: Arc, + worker_id: String, + lease: Duration, + max_attempts: u32, + batch_size: usize, + destination: Option, + buffer: VecDeque, +} + +impl OutboxSource +where + S: AsyncOutboxStore, +{ + /// Create a source. `worker_id` scopes claims; `max_attempts` is the + /// retryable-failure ceiling before a row is failed. + pub fn new(store: Arc, worker_id: impl Into, max_attempts: u32) -> Self { + Self { + store, + worker_id: worker_id.into(), + lease: DEFAULT_OUTBOX_SOURCE_LEASE, + max_attempts, + batch_size: DEFAULT_OUTBOX_SOURCE_BATCH, + destination: None, + buffer: VecDeque::new(), + } + } + + /// Set the claim lease / visibility timeout. + pub fn with_lease(mut self, lease: Duration) -> Self { + self.lease = lease; + self + } + + /// Set how many rows are claimed per refill. + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = batch_size; + self + } + + /// Only claim rows bound to this point-to-point destination. + pub fn with_destination(mut self, destination: impl Into) -> Self { + self.destination = Some(destination.into()); + self + } + + fn claim_request(&self) -> ClaimOutboxMessages { + let mut request = + ClaimOutboxMessages::new(self.worker_id.clone(), self.batch_size, self.lease); + if let Some(destination) = &self.destination { + request = request.to_destination(destination.clone()); + } + request + } +} + +impl AsyncMessageSource for OutboxSource +where + S: AsyncOutboxStore, +{ + type Received = ReceivedOutboxMessage; + + async fn recv(&mut self) -> Result, TransportError> { + if self.buffer.is_empty() { + let claimed = self.store.claim_async(self.claim_request()).await?; + self.buffer.extend(claimed); + } + match self.buffer.pop_front() { + Some(row) => { + let claim = OutboxClaimRef::from_message(&row)?; + Ok(Some(ReceivedOutboxMessage { + store: self.store.clone(), + message: Message::from(&row), + claim, + max_attempts: self.max_attempts, + })) + } + None => Ok(None), + } + } +} + +/// A claimed outbox row, settled back to the store on ack/nack. +pub struct ReceivedOutboxMessage { + store: Arc, + message: Message, + claim: OutboxClaimRef, + max_attempts: u32, +} + +impl ReceivedMessage for ReceivedOutboxMessage +where + S: AsyncOutboxStore, +{ + fn message(&self) -> &Message { + &self.message + } + + /// Complete the row (transport delivery succeeded). + async fn ack(self) -> Result<(), TransportError> { + self.store.complete_async(&self.claim).await?; + Ok(()) + } + + /// Release for retry, or fail once the attempt ceiling is reached. + async fn nack(self, reason: &str) -> Result<(), TransportError> { + self.store + .record_failure_async(&self.claim, reason, self.max_attempts) + .await?; + Ok(()) + } + + /// Fail the row terminally (the outbox `Failed` status is the DLQ/archive). + async fn dead_letter(self, reason: &str) -> Result<(), TransportError> { + self.store.fail_async(&self.claim, reason).await?; + Ok(()) + } + + /// Park terminally for manual inspection (same `Failed` status as dead-letter + /// in the outbox's state model). + async fn park(self, reason: &str) -> Result<(), TransportError> { + self.store.fail_async(&self.claim, reason).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::transport::{run_source, RunOptions}; + use crate::microsvc::Service; + use crate::{ + CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, OutboxStore, + TransactionalCommit, + }; + use serde_json::json; + use std::future::Future; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + fn store_row(repo: &HashMapRepository, id: &str, name: &str) { + let message = OutboxMessage::create(id, name, b"{}".to_vec()).unwrap(); + let mut batch = CommitBatch::empty(); + batch.outbox_messages.push(message); + repo.commit_batch(batch).unwrap(); + } + + fn status(repo: &HashMapRepository, id: &str) -> Option { + let store = repo.outbox_store(); + [ + OutboxMessageStatus::Pending, + OutboxMessageStatus::InFlight, + OutboxMessageStatus::Published, + OutboxMessageStatus::Failed, + ] + .into_iter() + .find(|status| { + store + .messages_by_status(status.clone()) + .unwrap() + .iter() + .any(|m| m.id() == id) + }) + } + + fn source(repo: &HashMapRepository) -> OutboxSource { + OutboxSource::new(Arc::new(repo.outbox_store()), "pg-transport", 3) + } + + #[test] + fn recv_yields_claimed_rows_then_drains_to_none() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "evt"); + store_row(&repo, "m2", "evt"); + let mut src = source(&repo); + + let first = block_on(src.recv()).unwrap().expect("first row"); + let second = block_on(src.recv()).unwrap().expect("second row"); + // Both claimed (in-flight) and held by this source; nothing else claimable. + let third = block_on(src.recv()).unwrap(); + assert!(third.is_none(), "drains to None once nothing is claimable"); + + let mut ids = vec![ + first.message().id().unwrap().to_string(), + second.message().id().unwrap().to_string(), + ]; + ids.sort(); + assert_eq!(ids, vec!["m1".to_string(), "m2".to_string()]); + } + + #[test] + fn ack_completes_the_row() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "evt"); + let mut src = source(&repo); + let received = block_on(src.recv()).unwrap().unwrap(); + block_on(received.ack()).unwrap(); + assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Published)); + } + + #[test] + fn nack_releases_for_retry() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "evt"); + let mut src = source(&repo); + let received = block_on(src.recv()).unwrap().unwrap(); + block_on(received.nack("transient")).unwrap(); + assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Pending)); + } + + #[test] + fn dead_letter_fails_the_row() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "evt"); + let mut src = source(&repo); + let received = block_on(src.recv()).unwrap().unwrap(); + block_on(received.dead_letter("poison")).unwrap(); + assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Failed)); + } + + #[test] + fn run_source_drains_outbox_and_completes() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "evt"); + store_row(&repo, "m2", "evt"); + + let handled = Arc::new(std::sync::Mutex::new(Vec::::new())); + let h = handled.clone(); + let service = Arc::new(Service::new(()).event("evt").handle(move |ctx| { + h.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })); + + block_on(run_source(service, source(&repo), RunOptions::idempotent())).unwrap(); + + let mut ids = handled.lock().unwrap().clone(); + ids.sort(); + assert_eq!(ids, vec!["m1".to_string(), "m2".to_string()]); + assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Published)); + assert_eq!(status(&repo, "m2"), Some(OutboxMessageStatus::Published)); + } + + #[test] + fn unhandled_outbox_message_is_acked_and_completed() { + let repo = HashMapRepository::new(); + store_row(&repo, "m1", "unrelated"); + // Service handles a different event; the unrelated row is acked-ignored, + // i.e. completed, so it does not loop forever. + let service: Arc> = + Arc::new(Service::new(()).event("evt").handle(|_| Ok(json!({})))); + block_on(run_source(service, source(&repo), RunOptions::idempotent())).unwrap(); + assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Published)); + } +} diff --git a/tests/postgres_transport/main.rs b/tests/postgres_transport/main.rs new file mode 100644 index 000000000..3b967bf1f --- /dev/null +++ b/tests/postgres_transport/main.rs @@ -0,0 +1,185 @@ +//! Postgres transport adapter integration tests. +//! +//! Exercises `OutboxSource` — the Postgres "starter" +//! durable transport — against a real Postgres: claim (`FOR UPDATE SKIP LOCKED` +//! with a lease), dispatch, and settle by row status. Skips when `DATABASE_URL` +//! is unset. +#![cfg(feature = "postgres")] + +#[path = "../support/postgres.rs"] +mod postgres; + +use std::sync::{Arc, Mutex}; + +use serde_json::json; +use sourced_rust::microsvc::transport::{ + run_source, AsyncMessageSource, OutboxSource, ReceivedMessage, RunOptions, +}; +use sourced_rust::microsvc::Service; +use sourced_rust::{ + AsyncCommitBatch, AsyncOutboxStore, AsyncTransactionalCommit, OutboxMessage, + OutboxMessageStatus, PostgresOutboxStore, PostgresRepository, +}; + +const SKIP: &str = "skipping postgres transport test"; + +async fn enqueue(repo: &PostgresRepository, id: &str, name: &str) { + let mut batch = AsyncCommitBatch::empty(); + batch + .outbox_messages + .push(OutboxMessage::create(id, name, b"{}".to_vec()).unwrap()); + repo.commit_batch_async(batch) + .await + .expect("outbox row should commit"); +} + +async fn status(store: &PostgresOutboxStore, id: &str) -> Option { + for s in [ + OutboxMessageStatus::Pending, + OutboxMessageStatus::InFlight, + OutboxMessageStatus::Published, + OutboxMessageStatus::Failed, + ] { + if store + .messages_by_status_async(s.clone()) + .await + .unwrap() + .iter() + .any(|m| m.id() == id) + { + return Some(s); + } + } + None +} + +fn recording_service(handled: Arc>>) -> Arc> { + Arc::new(Service::new(()).event("evt").handle(move |ctx| { + handled + .lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })) +} + +#[tokio::test] +async fn outbox_source_run_drains_and_completes() { + let Some(schema) = postgres::PostgresTestSchema::create_from_env("pg_tx_drain", SKIP).await + else { + return; + }; + let repo = schema.repository().await; + enqueue(&repo, "m1", "evt").await; + enqueue(&repo, "m2", "evt").await; + let store = Arc::new(repo.outbox_store()); + + let handled = Arc::new(Mutex::new(Vec::new())); + let service = recording_service(handled.clone()); + run_source( + service, + OutboxSource::new(store.clone(), "pg-drain", 3), + RunOptions::idempotent(), + ) + .await + .unwrap(); + + let mut ids = handled.lock().unwrap().clone(); + ids.sort(); + assert_eq!(ids, vec!["m1".to_string(), "m2".to_string()]); + assert_eq!( + status(&store, "m1").await, + Some(OutboxMessageStatus::Published) + ); + assert_eq!( + status(&store, "m2").await, + Some(OutboxMessageStatus::Published) + ); +} + +#[tokio::test] +async fn concurrent_sources_process_each_row_once() { + let Some(schema) = + postgres::PostgresTestSchema::create_from_env("pg_tx_concurrent", SKIP).await + else { + return; + }; + let repo = schema.repository().await; + let ids: Vec = (0..20).map(|i| format!("c{i}")).collect(); + for id in &ids { + enqueue(&repo, id, "evt").await; + } + let store = Arc::new(repo.outbox_store()); + + let handled = Arc::new(Mutex::new(Vec::new())); + let run = |worker: &'static str| { + run_source( + recording_service(handled.clone()), + OutboxSource::new(store.clone(), worker, 3), + RunOptions::idempotent(), + ) + }; + // Two competing consumers drain concurrently; SKIP LOCKED guarantees each + // row is claimed (and handled) exactly once. + let (a, b) = tokio::join!(run("worker-a"), run("worker-b")); + a.unwrap(); + b.unwrap(); + + let mut got = handled.lock().unwrap().clone(); + got.sort(); + let unique = { + let mut u = got.clone(); + u.dedup(); + u + }; + assert_eq!(got, unique, "no row handled more than once"); + assert_eq!(unique.len(), ids.len(), "every row handled"); +} + +#[tokio::test] +async fn nack_releases_then_a_later_claim_completes() { + let Some(schema) = postgres::PostgresTestSchema::create_from_env("pg_tx_retry", SKIP).await + else { + return; + }; + let repo = schema.repository().await; + enqueue(&repo, "m1", "evt").await; + let store = Arc::new(repo.outbox_store()); + + // First claim, nack -> released to pending (attempts incremented). + let mut source = OutboxSource::new(store.clone(), "pg-retry", 5); + let received = source.recv().await.unwrap().expect("a claimable row"); + received.nack("transient").await.unwrap(); + assert_eq!( + status(&store, "m1").await, + Some(OutboxMessageStatus::Pending) + ); + + // A later claim completes it. + let mut source2 = OutboxSource::new(store.clone(), "pg-retry-2", 5); + let received2 = source2.recv().await.unwrap().expect("a re-claimable row"); + received2.ack().await.unwrap(); + assert_eq!( + status(&store, "m1").await, + Some(OutboxMessageStatus::Published) + ); +} + +#[tokio::test] +async fn dead_letter_marks_row_failed() { + let Some(schema) = postgres::PostgresTestSchema::create_from_env("pg_tx_dlq", SKIP).await + else { + return; + }; + let repo = schema.repository().await; + enqueue(&repo, "m1", "evt").await; + let store = Arc::new(repo.outbox_store()); + + let mut source = OutboxSource::new(store.clone(), "pg-dlq", 3); + let received = source.recv().await.unwrap().expect("a claimable row"); + received.dead_letter("poison").await.unwrap(); + assert_eq!( + status(&store, "m1").await, + Some(OutboxMessageStatus::Failed) + ); +} From b7ca6924e56ae05f9f1677971229bf05bfc7ad85 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:03:19 -0500 Subject: [PATCH 05/39] feat(transport): NATS JetStream adapter + integration tests + CI NatsPublisher (publish ack threshold) and NatsJetStreamSource (durable pull consumer; ack/nak/term settle) behind the nats feature, over the shared AsyncMessagePublisher/AsyncMessageSource/run_source boundary. Stable id + metadata ride as headers (Nats-Msg-Id is also the JetStream dedup key). Adds tests/nats_transport (verified against nats:2.10 -js: round-trip + metadata preservation), a nats CI job, and the nats compose service. Implements [[tasks/nats-transport-adapter]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/on-pr-quality.yaml | 25 +++ Cargo.toml | 3 + src/microsvc/transport/mod.rs | 5 + src/microsvc/transport/nats.rs | 274 +++++++++++++++++++++++++++ tests/nats_transport/main.rs | 129 +++++++++++++ 5 files changed, 436 insertions(+) create mode 100644 src/microsvc/transport/nats.rs create mode 100644 tests/nats_transport/main.rs diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index 7534ad827..6b9d589d1 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -44,3 +44,28 @@ jobs: cargo test --test postgres_repository_conformance --all-features --verbose cargo test --test postgres_transport --all-features --verbose cargo test --test distributed_read_model --all-features --verbose + + nats: + name: NATS JetStream Integration Tests + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + NATS_URL: nats://localhost:4222 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + # NATS service containers can't override the entrypoint to enable + # JetStream, so start it as a step instead. + - name: Start NATS (JetStream) + run: | + docker run -d --name nats -p 4222:4222 nats:2.10-alpine -js + for i in $(seq 1 30); do + if nc -z localhost 4222; then echo "nats up"; break; fi + sleep 1 + done + - name: Run NATS transport integration tests + run: cargo test --test nats_transport --features nats --verbose diff --git a/Cargo.toml b/Cargo.toml index 68e7fdb85..ed5c5d0b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,10 +34,13 @@ http = ["bus", "dep:axum", "dep:tokio"] grpc = ["bus", "dep:tonic", "dep:prost", "dep:tokio"] postgres = ["dep:sqlx", "dep:tokio", "sqlx/postgres", "sqlx/runtime-tokio"] sqlite = ["dep:sqlx", "dep:tokio", "sqlx/runtime-tokio", "sqlx/sqlite"] +nats = ["dep:async-nats", "dep:futures", "dep:tokio"] [dependencies] +async-nats = { version = "0.38", optional = true } axum = { version = "0.7", optional = true } base64 = "0.22.1" +futures = { version = "0.3", optional = true } bitcode = { version = "0.6.9", features = ["serde"] } event-emitter-rs = { version = "0.1.4", optional = true } serde = { version = "1.0.210", features = ["derive"] } diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 7548a7a47..35bba6246 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -81,6 +81,8 @@ use crate::microsvc::Message; mod capabilities; mod error; mod failure_policy; +#[cfg(feature = "nats")] +mod nats; mod outbox_dispatch; mod outbox_source; mod publisher; @@ -89,6 +91,9 @@ mod runner; mod source; mod stable_id; +#[cfg(feature = "nats")] +pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; + pub use capabilities::{ConsumerAckKind, KnativeIntegrationKind, TransportCapabilities}; pub use error::{TransportError, TransportErrorKind}; pub use failure_policy::{FailureAction, FailurePolicy}; diff --git a/src/microsvc/transport/nats.rs b/src/microsvc/transport/nats.rs new file mode 100644 index 000000000..3343bc22e --- /dev/null +++ b/src/microsvc/transport/nats.rs @@ -0,0 +1,274 @@ +//! NATS JetStream transport adapter. +//! +//! Maps the canonical [`Message`] onto NATS JetStream: [`NatsPublisher`] publishes +//! to a subject (waiting for the JetStream publish ack — the durable publish +//! threshold), and [`NatsJetStreamSource`] pulls from a durable consumer and +//! settles via JetStream ack semantics (ack→`Ack`, nack→`Nak`, dead-letter/park→ +//! `Term`). The stable message id rides as the `Nats-Msg-Id` header so JetStream +//! dedup and downstream `Message.id` agree. +//! +//! Requires the `nats` feature. Integration-tested in `tests/nats_transport` +//! against a JetStream-enabled server (see `compose.yaml`). + +use std::time::Duration; + +use async_nats::jetstream::consumer::pull::Config as PullConfig; +use async_nats::jetstream::consumer::Consumer; +use async_nats::jetstream::stream::Config as StreamConfig; +use async_nats::jetstream::{self, AckKind}; +use futures::StreamExt; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{AsyncMessagePublisher, TransportError}; +use crate::microsvc::{Message, MessageKind}; + +/// Header carrying the stable message id (and JetStream dedup key). +const MESSAGE_ID_HEADER: &str = "Nats-Msg-Id"; +/// Header carrying the canonical message kind. +const MESSAGE_KIND_HEADER: &str = "X-Sourced-Kind"; + +fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { + TransportError::retryable(format!("{context}: {err}")) +} + +/// Publishes canonical messages to a NATS JetStream subject. +/// +/// The subject defaults to the message name; override with [`with_subject_prefix`] +/// to publish to `{prefix}.{name}`. +/// +/// [`with_subject_prefix`]: NatsPublisher::with_subject_prefix +pub struct NatsPublisher { + jetstream: jetstream::Context, + subject_prefix: Option, +} + +impl NatsPublisher { + /// Create a publisher over an existing JetStream context. + pub fn new(jetstream: jetstream::Context) -> Self { + Self { + jetstream, + subject_prefix: None, + } + } + + /// Connect to a NATS server URL and create a JetStream publisher. + pub async fn connect(url: &str) -> Result { + let client = async_nats::connect(url) + .await + .map_err(|err| retryable("nats connect", err))?; + Ok(Self::new(jetstream::new(client))) + } + + /// Publish to `{prefix}.{message.name}` instead of `{message.name}`. + pub fn with_subject_prefix(mut self, prefix: impl Into) -> Self { + self.subject_prefix = Some(prefix.into()); + self + } + + fn subject(&self, message: &Message) -> String { + match &self.subject_prefix { + Some(prefix) => format!("{prefix}.{}", message.name()), + None => message.name().to_string(), + } + } +} + +impl AsyncMessagePublisher for NatsPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + let subject = self.subject(&message); + let mut headers = async_nats::HeaderMap::new(); + if let Some(id) = message.id() { + headers.insert(MESSAGE_ID_HEADER, id); + } + headers.insert(MESSAGE_KIND_HEADER, kind_str(message.kind)); + for (key, value) in &message.metadata { + headers.insert(key.as_str(), value.as_str()); + } + + // Publish ack (the durable publish threshold): both awaits must succeed. + let ack_future = self + .jetstream + .publish_with_headers(subject, headers, message.payload.clone().into()) + .await + .map_err(|err| retryable("nats publish", err))?; + ack_future + .await + .map_err(|err| retryable("nats publish ack", err))?; + Ok(()) + } +} + +/// A pull-based JetStream source bound to a durable consumer. +pub struct NatsJetStreamSource { + consumer: Consumer, + fetch_timeout: Duration, +} + +impl NatsJetStreamSource { + /// Wrap an existing durable pull consumer. + pub fn new(consumer: Consumer) -> Self { + Self { + consumer, + fetch_timeout: Duration::from_millis(500), + } + } + + /// How long `recv` waits for a message before returning `Ok(None)`. + pub fn with_fetch_timeout(mut self, timeout: Duration) -> Self { + self.fetch_timeout = timeout; + self + } + + /// Connect to a NATS server URL, then create/open the stream + consumer. + pub async fn connect( + url: &str, + stream_name: &str, + subjects: Vec, + durable: &str, + ) -> Result { + let client = async_nats::connect(url) + .await + .map_err(|err| retryable("nats connect", err))?; + let jetstream = jetstream::new(client); + Self::from_context(&jetstream, stream_name, subjects, durable).await + } + + /// Create or open a JetStream stream + durable pull consumer, then a source. + /// + /// `subjects` binds the stream; `durable` names the consumer so progress + /// survives restarts. + pub async fn from_context( + jetstream: &jetstream::Context, + stream_name: &str, + subjects: Vec, + durable: &str, + ) -> Result { + let stream = jetstream + .get_or_create_stream(StreamConfig { + name: stream_name.to_string(), + subjects, + ..Default::default() + }) + .await + .map_err(|err| retryable("nats get_or_create_stream", err))?; + let consumer = stream + .get_or_create_consumer( + durable, + PullConfig { + durable_name: Some(durable.to_string()), + ..Default::default() + }, + ) + .await + .map_err(|err| retryable("nats get_or_create_consumer", err))?; + Ok(Self::new(consumer)) + } +} + +impl AsyncMessageSource for NatsJetStreamSource { + type Received = NatsReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let mut batch = self + .consumer + .batch() + .max_messages(1) + .expires(self.fetch_timeout) + .messages() + .await + .map_err(|err| retryable("nats fetch", err))?; + + match batch.next().await { + Some(Ok(message)) => Ok(Some(NatsReceived::from_jetstream(message))), + Some(Err(err)) => Err(retryable("nats batch message", err)), + None => Ok(None), + } + } +} + +/// A JetStream message plus the means to ack/nak/term it. +pub struct NatsReceived { + raw: jetstream::Message, + message: Message, +} + +impl NatsReceived { + fn from_jetstream(raw: jetstream::Message) -> Self { + let name = raw.subject.to_string(); + let payload = raw.payload.to_vec(); + let mut id = None; + let mut kind = MessageKind::Event; + let mut metadata = Vec::new(); + if let Some(headers) = raw.headers.as_ref() { + for (key, values) in headers.iter() { + let key = key.to_string(); + if let Some(value) = values.last() { + let value = value.to_string(); + match key.as_str() { + MESSAGE_ID_HEADER => id = Some(value), + MESSAGE_KIND_HEADER => kind = kind_from_str(&value), + _ => metadata.push((key, value)), + } + } + } + } + let mut message = Message::new(name, kind, payload); + message.id = id; + message.metadata = metadata; + Self { raw, message } + } + + async fn settle(self, kind: AckKind) -> Result<(), TransportError> { + match kind { + AckKind::Ack => self + .raw + .ack() + .await + .map_err(|err| retryable("nats ack", err)), + other => self + .raw + .ack_with(other) + .await + .map_err(|err| retryable("nats ack_with", err)), + } + } +} + +impl ReceivedMessage for NatsReceived { + fn message(&self) -> &Message { + &self.message + } + + async fn ack(self) -> Result<(), TransportError> { + self.settle(AckKind::Ack).await + } + + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + // Nak with no delay: JetStream redelivers per the consumer policy. + self.settle(AckKind::Nak(None)).await + } + + async fn dead_letter(self, _reason: &str) -> Result<(), TransportError> { + // Term: stop redelivery. A real DLQ bridge can subscribe to the stream's + // advisory/max-deliver subjects; Term is the "do not redeliver" signal. + self.settle(AckKind::Term).await + } + + async fn park(self, _reason: &str) -> Result<(), TransportError> { + self.settle(AckKind::Term).await + } +} + +fn kind_str(kind: MessageKind) -> &'static str { + match kind { + MessageKind::Command => "command", + MessageKind::Event => "event", + } +} + +fn kind_from_str(value: &str) -> MessageKind { + match value { + "command" => MessageKind::Command, + _ => MessageKind::Event, + } +} diff --git a/tests/nats_transport/main.rs b/tests/nats_transport/main.rs new file mode 100644 index 000000000..9060cc69b --- /dev/null +++ b/tests/nats_transport/main.rs @@ -0,0 +1,129 @@ +//! NATS JetStream transport adapter integration tests. +//! +//! Publishes via `NatsPublisher` and consumes via `NatsJetStreamSource` against a +//! JetStream-enabled NATS server. Skips when `NATS_URL` is unset. +#![cfg(feature = "nats")] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; +use sourced_rust::microsvc::transport::{ + run_source, AsyncMessagePublisher, NatsJetStreamSource, NatsPublisher, RunOptions, +}; +use sourced_rust::microsvc::{Message, MessageKind, Service}; + +static SEQ: AtomicU64 = AtomicU64::new(1); + +fn nats_url() -> Option { + match std::env::var("NATS_URL") { + Ok(url) => Some(url), + Err(_) => { + eprintln!("skipping nats transport test: NATS_URL is not set"); + None + } + } +} + +/// Unique subject/stream/durable per test so JetStream state does not collide. +fn unique(prefix: &str) -> String { + format!("{prefix}_{}", SEQ.fetch_add(1, Ordering::SeqCst)) +} + +#[tokio::test] +async fn publish_then_consume_round_trips_through_jetstream() { + let Some(url) = nats_url() else { return }; + let subject = unique("evt"); + let stream = unique("STREAM"); + let durable = unique("consumer"); + + // Create the stream + durable consumer first so the stream exists before we + // publish (JetStream publish requires a stream bound to the subject). + let source = NatsJetStreamSource::connect(&url, &stream, vec![subject.clone()], &durable) + .await + .expect("connect source") + .with_fetch_timeout(Duration::from_millis(800)); + + // Publish three events. + let publisher = NatsPublisher::connect(&url) + .await + .expect("connect publisher"); + for i in 0..3 { + let message = + Message::new(&subject, MessageKind::Event, b"{}".to_vec()).with_id(format!("m{i}")); + publisher.publish(message).await.expect("publish"); + } + + // Consume via the shared runner. + let handled = Arc::new(Mutex::new(Vec::::new())); + let h = handled.clone(); + let subject_for_handler = subject.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(subject.clone().into_boxed_str())) + .handle(move |ctx| { + assert_eq!(ctx.message().name(), subject_for_handler); + h.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + }), + ); + + run_source(service, source, RunOptions::idempotent()) + .await + .expect("run_source drains the stream"); + + let mut ids = handled.lock().unwrap().clone(); + ids.sort(); + assert_eq!( + ids, + vec!["m0".to_string(), "m1".to_string(), "m2".to_string()] + ); +} + +#[tokio::test] +async fn message_id_and_metadata_survive_the_round_trip() { + let Some(url) = nats_url() else { return }; + let subject = unique("evt"); + let stream = unique("STREAM"); + let durable = unique("consumer"); + + let source = NatsJetStreamSource::connect(&url, &stream, vec![subject.clone()], &durable) + .await + .expect("connect source") + .with_fetch_timeout(Duration::from_millis(800)); + + let publisher = NatsPublisher::connect(&url) + .await + .expect("connect publisher"); + let message = Message::new(&subject, MessageKind::Event, br#"{"k":"v"}"#.to_vec()) + .with_id("evt-1") + .with_metadata("correlation_id", "corr-9"); + publisher.publish(message).await.expect("publish"); + + let observed = Arc::new(Mutex::new(None)); + let o = observed.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(subject.clone().into_boxed_str())) + .handle(move |ctx| { + let m = ctx.message(); + *o.lock().unwrap() = Some(( + m.id().map(str::to_string), + m.correlation_id().map(str::to_string), + m.payload().to_vec(), + )); + Ok(json!({})) + }), + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + + let got = observed.lock().unwrap().clone().expect("handler ran"); + assert_eq!(got.0.as_deref(), Some("evt-1")); + assert_eq!(got.1.as_deref(), Some("corr-9")); + assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); +} From a5b9a8b398ff29a3de9f92a1c31215b20ebb5d71 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:11:43 -0500 Subject: [PATCH 06/39] feat(transport): RabbitMQ (AMQP) adapter + integration tests + CI RabbitPublisher (publisher-confirm threshold) and RabbitSource (basic_get; ack/nack-requeue/reject settle) behind the rabbitmq feature, over the shared transport traits. Stable id via the AMQP message_id property, metadata+kind via headers. Adds tests/rabbitmq_transport (verified against rabbitmq:3.13), a rabbitmq CI job (service container), and updates the module docs for the NATS/RabbitMQ adapters. Implements [[tasks/rabbitmq-transport-adapter]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/on-pr-quality.yaml | 26 +++ Cargo.toml | 2 + src/microsvc/transport/mod.rs | 11 +- src/microsvc/transport/rabbitmq.rs | 248 +++++++++++++++++++++++++++ tests/rabbitmq_transport/main.rs | 118 +++++++++++++ 5 files changed, 403 insertions(+), 2 deletions(-) create mode 100644 src/microsvc/transport/rabbitmq.rs create mode 100644 tests/rabbitmq_transport/main.rs diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index 6b9d589d1..f10b17213 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -69,3 +69,29 @@ jobs: done - name: Run NATS transport integration tests run: cargo test --test nats_transport --features nats --verbose + + rabbitmq: + name: RabbitMQ Integration Tests + runs-on: ubuntu-latest + services: + rabbitmq: + image: rabbitmq:3.13-management-alpine + ports: + - 5672:5672 + options: >- + --health-cmd "rabbitmq-diagnostics -q ping" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + CARGO_TERM_COLOR: always + AMQP_URL: amqp://guest:guest@localhost:5672/%2f + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + - name: Run RabbitMQ transport integration tests + run: cargo test --test rabbitmq_transport --features rabbitmq --verbose diff --git a/Cargo.toml b/Cargo.toml index ed5c5d0b7..a825f6a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,12 +35,14 @@ grpc = ["bus", "dep:tonic", "dep:prost", "dep:tokio"] postgres = ["dep:sqlx", "dep:tokio", "sqlx/postgres", "sqlx/runtime-tokio"] sqlite = ["dep:sqlx", "dep:tokio", "sqlx/runtime-tokio", "sqlx/sqlite"] nats = ["dep:async-nats", "dep:futures", "dep:tokio"] +rabbitmq = ["dep:lapin", "dep:futures", "dep:tokio"] [dependencies] async-nats = { version = "0.38", optional = true } axum = { version = "0.7", optional = true } base64 = "0.22.1" futures = { version = "0.3", optional = true } +lapin = { version = "2", optional = true } bitcode = { version = "0.6.9", features = ["serde"] } event-emitter-rs = { version = "0.1.4", optional = true } serde = { version = "1.0.210", features = ["derive"] } diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 35bba6246..6da18985a 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -21,8 +21,11 @@ //! `Message` and dispatch them, sharing one claim → publish → complete path //! between background polling and after-commit immediate dispatch. //! -//! The Knative/HTTP ingress shape and the concrete broker adapters are still -//! separate slices. Everything builds on the vocabulary defined here: +//! Concrete adapters build on these traits: the Postgres outbox-backed source +//! ([`OutboxSource`]) is always available; the NATS JetStream and RabbitMQ +//! adapters are behind the `nats` and `rabbitmq` features. The Knative/HTTP +//! ingress shape and the Kafka adapter are still separate slices. Everything +//! builds on the vocabulary defined here: //! //! - [`TransportError`] / [`TransportErrorKind`] — retryable vs permanent //! classification the runner uses to decide between redelivery and the @@ -86,6 +89,8 @@ mod nats; mod outbox_dispatch; mod outbox_source; mod publisher; +#[cfg(feature = "rabbitmq")] +mod rabbitmq; mod run_options; mod runner; mod source; @@ -93,6 +98,8 @@ mod stable_id; #[cfg(feature = "nats")] pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; +#[cfg(feature = "rabbitmq")] +pub use rabbitmq::{RabbitPublisher, RabbitReceived, RabbitSource}; pub use capabilities::{ConsumerAckKind, KnativeIntegrationKind, TransportCapabilities}; pub use error::{TransportError, TransportErrorKind}; diff --git a/src/microsvc/transport/rabbitmq.rs b/src/microsvc/transport/rabbitmq.rs new file mode 100644 index 000000000..6c4dc622a --- /dev/null +++ b/src/microsvc/transport/rabbitmq.rs @@ -0,0 +1,248 @@ +//! RabbitMQ (AMQP 0-9-1) transport adapter. +//! +//! [`RabbitPublisher`] publishes a canonical [`Message`] to the default exchange +//! keyed by the message name, waiting for the **publisher confirm** (the durable +//! publish threshold). [`RabbitSource`] polls a queue with `basic_get` and +//! settles via AMQP ack/nack/reject (ack→ack, nack→nack+requeue, +//! dead-letter/park→reject without requeue, which routes to a dead-letter +//! exchange when one is configured on the queue). +//! +//! Requires the `rabbitmq` feature. Integration-tested in +//! `tests/rabbitmq_transport` against a broker (see `compose.yaml`). + +use lapin::message::Delivery; +use lapin::options::{ + BasicAckOptions, BasicGetOptions, BasicNackOptions, BasicPublishOptions, BasicRejectOptions, + ConfirmSelectOptions, QueueDeclareOptions, +}; +use lapin::types::{AMQPValue, FieldTable, ShortString}; +use lapin::{BasicProperties, Channel, Connection, ConnectionProperties}; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{AsyncMessagePublisher, TransportError}; +use crate::microsvc::{Message, MessageKind}; + +const MESSAGE_KIND_HEADER: &str = "x-sourced-kind"; + +fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { + TransportError::retryable(format!("{context}: {err}")) +} + +async fn connect_channel(uri: &str) -> Result { + let connection = Connection::connect(uri, ConnectionProperties::default()) + .await + .map_err(|err| retryable("amqp connect", err))?; + connection + .create_channel() + .await + .map_err(|err| retryable("amqp channel", err)) +} + +/// Publishes canonical messages to the default exchange, keyed by message name. +pub struct RabbitPublisher { + channel: Channel, +} + +impl RabbitPublisher { + /// Wrap an existing channel (publisher confirms are enabled on connect). + pub fn new(channel: Channel) -> Self { + Self { channel } + } + + /// Connect to an AMQP URI and enable publisher confirms. + pub async fn connect(uri: &str) -> Result { + let channel = connect_channel(uri).await?; + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|err| retryable("amqp confirm_select", err))?; + Ok(Self::new(channel)) + } +} + +fn message_properties(message: &Message) -> BasicProperties { + let mut headers = FieldTable::default(); + headers.insert( + ShortString::from(MESSAGE_KIND_HEADER), + AMQPValue::LongString(kind_str(message.kind).into()), + ); + for (key, value) in &message.metadata { + headers.insert( + ShortString::from(key.as_str()), + AMQPValue::LongString(value.as_str().into()), + ); + } + let mut properties = BasicProperties::default().with_headers(headers); + if let Some(id) = message.id() { + properties = properties.with_message_id(ShortString::from(id)); + } + properties +} + +impl AsyncMessagePublisher for RabbitPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + let confirm = self + .channel + .basic_publish( + "", // default exchange: routes to the queue named by the routing key + message.name(), + BasicPublishOptions::default(), + &message.payload, + message_properties(&message), + ) + .await + .map_err(|err| retryable("amqp publish", err))?; + let confirmation = confirm + .await + .map_err(|err| retryable("amqp publisher confirm", err))?; + if confirmation.is_nack() { + return Err(TransportError::retryable("amqp publisher confirm: nack")); + } + Ok(()) + } +} + +/// Polls a single queue with `basic_get`. +pub struct RabbitSource { + channel: Channel, + queue: String, +} + +impl RabbitSource { + /// Wrap an existing channel bound to `queue`. + pub fn new(channel: Channel, queue: impl Into) -> Self { + Self { + channel, + queue: queue.into(), + } + } + + /// Connect, declare a durable queue, and poll it. The default exchange routes + /// a message published with routing key == `queue` into this queue, so the + /// queue name is the message name consumers subscribe to. + pub async fn connect(uri: &str, queue: &str) -> Result { + let channel = connect_channel(uri).await?; + channel + .queue_declare( + queue, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|err| retryable("amqp queue_declare", err))?; + Ok(Self::new(channel, queue)) + } +} + +impl AsyncMessageSource for RabbitSource { + type Received = RabbitReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let message = self + .channel + .basic_get(&self.queue, BasicGetOptions::default()) + .await + .map_err(|err| retryable("amqp basic_get", err))?; + Ok(message.map(|get| RabbitReceived::from_delivery(get.delivery, self.queue.clone()))) + } +} + +/// An AMQP delivery plus its settle handle. +pub struct RabbitReceived { + delivery: Delivery, + message: Message, +} + +impl RabbitReceived { + fn from_delivery(delivery: Delivery, queue: String) -> Self { + let payload = delivery.data.clone(); + let id = delivery + .properties + .message_id() + .as_ref() + .map(|s| s.to_string()); + let mut kind = MessageKind::Event; + let mut metadata = Vec::new(); + if let Some(headers) = delivery.properties.headers().as_ref() { + for (key, value) in headers.inner() { + let key = key.to_string(); + let value = amqp_value_to_string(value); + if key == MESSAGE_KIND_HEADER { + kind = kind_from_str(&value); + } else { + metadata.push((key, value)); + } + } + } + // The queue name is the routed message name. + let mut message = Message::new(queue, kind, payload); + message.id = id; + message.metadata = metadata; + Self { delivery, message } + } +} + +impl ReceivedMessage for RabbitReceived { + fn message(&self) -> &Message { + &self.message + } + + async fn ack(self) -> Result<(), TransportError> { + self.delivery + .ack(BasicAckOptions::default()) + .await + .map_err(|err| retryable("amqp ack", err)) + } + + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + // Requeue for redelivery. + self.delivery + .nack(BasicNackOptions { + requeue: true, + ..Default::default() + }) + .await + .map_err(|err| retryable("amqp nack", err)) + } + + async fn dead_letter(self, _reason: &str) -> Result<(), TransportError> { + // Reject without requeue: routes to the queue's dead-letter exchange if + // one is configured, otherwise drops. + self.delivery + .reject(BasicRejectOptions { requeue: false }) + .await + .map_err(|err| retryable("amqp reject", err)) + } + + async fn park(self, _reason: &str) -> Result<(), TransportError> { + self.delivery + .reject(BasicRejectOptions { requeue: false }) + .await + .map_err(|err| retryable("amqp reject", err)) + } +} + +fn amqp_value_to_string(value: &AMQPValue) -> String { + match value { + AMQPValue::LongString(s) => s.to_string(), + AMQPValue::ShortString(s) => s.to_string(), + other => format!("{other:?}"), + } +} + +fn kind_str(kind: MessageKind) -> &'static str { + match kind { + MessageKind::Command => "command", + MessageKind::Event => "event", + } +} + +fn kind_from_str(value: &str) -> MessageKind { + match value { + "command" => MessageKind::Command, + _ => MessageKind::Event, + } +} diff --git a/tests/rabbitmq_transport/main.rs b/tests/rabbitmq_transport/main.rs new file mode 100644 index 000000000..2762a165a --- /dev/null +++ b/tests/rabbitmq_transport/main.rs @@ -0,0 +1,118 @@ +//! RabbitMQ transport adapter integration tests. +//! +//! Publishes via `RabbitPublisher` (publisher confirms) and consumes via +//! `RabbitSource` (`basic_get`) against a broker. Skips when `AMQP_URL` is unset. +#![cfg(feature = "rabbitmq")] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use serde_json::json; +use sourced_rust::microsvc::transport::{ + run_source, AsyncMessagePublisher, RabbitPublisher, RabbitSource, RunOptions, +}; +use sourced_rust::microsvc::{Message, MessageKind, Service}; + +static SEQ: AtomicU64 = AtomicU64::new(1); + +fn amqp_url() -> Option { + match std::env::var("AMQP_URL") { + Ok(url) => Some(url), + Err(_) => { + eprintln!("skipping rabbitmq transport test: AMQP_URL is not set"); + None + } + } +} + +fn unique(prefix: &str) -> String { + format!("{prefix}_{}", SEQ.fetch_add(1, Ordering::SeqCst)) +} + +#[tokio::test] +async fn publish_then_consume_round_trips_through_rabbitmq() { + let Some(url) = amqp_url() else { return }; + let queue = unique("evt"); + + // Declare the queue (source) before publishing so the default-exchange route + // has a destination. + let source = RabbitSource::connect(&url, &queue) + .await + .expect("connect source"); + let publisher = RabbitPublisher::connect(&url) + .await + .expect("connect publisher"); + for i in 0..3 { + let message = + Message::new(&queue, MessageKind::Event, b"{}".to_vec()).with_id(format!("m{i}")); + publisher.publish(message).await.expect("publish"); + } + + let handled = Arc::new(Mutex::new(Vec::::new())); + let h = handled.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(queue.clone().into_boxed_str())) + .handle(move |ctx| { + h.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + }), + ); + run_source(service, source, RunOptions::idempotent()) + .await + .expect("run_source drains the queue"); + + let mut ids = handled.lock().unwrap().clone(); + ids.sort(); + assert_eq!( + ids, + vec!["m0".to_string(), "m1".to_string(), "m2".to_string()] + ); +} + +#[tokio::test] +async fn message_id_and_metadata_survive_the_round_trip() { + let Some(url) = amqp_url() else { return }; + let queue = unique("evt"); + + let source = RabbitSource::connect(&url, &queue) + .await + .expect("connect source"); + let publisher = RabbitPublisher::connect(&url) + .await + .expect("connect publisher"); + publisher + .publish( + Message::new(&queue, MessageKind::Event, br#"{"k":"v"}"#.to_vec()) + .with_id("evt-1") + .with_metadata("correlation_id", "corr-9"), + ) + .await + .expect("publish"); + + let observed = Arc::new(Mutex::new(None)); + let o = observed.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(queue.clone().into_boxed_str())) + .handle(move |ctx| { + let m = ctx.message(); + *o.lock().unwrap() = Some(( + m.id().map(str::to_string), + m.correlation_id().map(str::to_string), + m.payload().to_vec(), + )); + Ok(json!({})) + }), + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + + let got = observed.lock().unwrap().clone().expect("handler ran"); + assert_eq!(got.0.as_deref(), Some("evt-1")); + assert_eq!(got.1.as_deref(), Some("corr-9")); + assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); +} From 512b688ee4f54081b24595b7833d067795cca695 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:16:49 -0500 Subject: [PATCH 07/39] feat(transport): Knative CloudEvents HTTP ingress cloud_events_router parses binary + structured CloudEvents into the canonical Message and calls Service::dispatch_message (the same boundary as run_source). HTTP response is the ack: 200 success, 503 retryable, 422 permanent, 400 malformed. knative_triggers() renders Trigger YAML from subscription_plan(). Retry/DLQ is platform-managed by Knative here (not this crate's FailurePolicy). Adds tests/knative_cloudevents (6 in-process HTTP integration tests). Behind the http feature. Implements [[tasks/knative-cloudevents-ingress]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/knative.rs | 286 ++++++++++++++++++++++++++++++ src/microsvc/transport/mod.rs | 4 + tests/knative_cloudevents/main.rs | 141 +++++++++++++++ 3 files changed, 431 insertions(+) create mode 100644 src/microsvc/transport/knative.rs create mode 100644 tests/knative_cloudevents/main.rs diff --git a/src/microsvc/transport/knative.rs b/src/microsvc/transport/knative.rs new file mode 100644 index 000000000..599cd1b4b --- /dev/null +++ b/src/microsvc/transport/knative.rs @@ -0,0 +1,286 @@ +//! Knative / CloudEvents HTTP ingress. +//! +//! Knative is *endpoint-driven*: the platform invokes an HTTP route, so this is +//! a separate ingress shape from the pull-based [`AsyncMessageSource`]. It is +//! NOT modeled as a polling source. The route parses a CloudEvent (binary or +//! structured HTTP mode), maps it to the canonical [`Message`], and calls the +//! same [`Service::dispatch_message`] boundary the direct-transport runner uses. +//! +//! Acknowledgement is the HTTP response: +//! +//! - handler success → `200 OK` (the only ack); +//! - retryable failure → `503` so Knative redelivers per its Delivery config; +//! - permanent failure → `422` so Knative stops retrying / dead-letters per its +//! Delivery config. +//! +//! Retry, backoff, and dead-lettering are **platform-managed** by Knative +//! Eventing here — unlike direct transports, where this crate's +//! [`FailurePolicy`](super::FailurePolicy) owns them. +//! +//! Requires the `http` feature. + +use std::sync::Arc; + +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::{Json, Router}; +use base64::Engine; +use serde_json::{json, Value}; + +use super::TransportError; +use crate::microsvc::{Message, MessageKind, Service, SubscriptionPlan}; + +const STRUCTURED_CONTENT_TYPE: &str = "application/cloudevents+json"; + +/// Build an axum router exposing a CloudEvents ingress at `POST /`. +/// +/// Compose it with other routes or serve it directly; Knative Triggers point a +/// subscriber `ref` at this endpoint. +pub fn cloud_events_router(service: Arc>) -> Router { + Router::new() + .route("/", axum::routing::post(ingress_handler)) + .with_state(service) +} + +async fn ingress_handler( + State(service): State>>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let message = match parse_cloud_event(&headers, &body) { + Ok(message) => message, + Err(reason) => return (StatusCode::BAD_REQUEST, reason).into_response(), + }; + + match service.dispatch_message(&message) { + Ok(value) => (StatusCode::OK, Json(value)).into_response(), + Err(err) => { + // Map our retryable/permanent classification onto HTTP so Knative's + // platform-managed retry/DLQ does the right thing. + let status = if TransportError::classify_handler_error(&err).is_retryable() { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::UNPROCESSABLE_ENTITY + }; + (status, Json(json!({ "error": err.to_string() }))).into_response() + } + } +} + +fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name).and_then(|value| value.to_str().ok()) +} + +/// Parse a CloudEvent in binary or structured HTTP mode into a [`Message`]. +fn parse_cloud_event(headers: &HeaderMap, body: &Bytes) -> Result { + let content_type = header(headers, "content-type").unwrap_or(""); + if content_type.starts_with(STRUCTURED_CONTENT_TYPE) { + parse_structured(body) + } else { + parse_binary(headers, body) + } +} + +/// Binary mode: attributes are `ce-*` headers, the body is the data. +fn parse_binary(headers: &HeaderMap, body: &Bytes) -> Result { + let id = header(headers, "ce-id").ok_or("missing ce-id header")?; + let name = header(headers, "ce-type").ok_or("missing ce-type header")?; + let content_type = header(headers, "content-type") + .unwrap_or("application/json") + .to_string(); + + let mut metadata = Vec::new(); + for (key, value) in headers.iter() { + let key = key.as_str(); + if let Some(attr) = key.strip_prefix("ce-") { + if attr == "id" || attr == "type" { + continue; + } + if let Ok(value) = value.to_str() { + metadata.push((attr.to_string(), value.to_string())); + } + } + } + + Ok(Message { + id: Some(id.to_string()), + name: name.to_string(), + kind: MessageKind::Event, + payload: body.to_vec(), + content_type, + metadata, + }) +} + +/// Structured mode: a single `application/cloudevents+json` document. +fn parse_structured(body: &Bytes) -> Result { + let event: Value = + serde_json::from_slice(body).map_err(|e| format!("invalid cloudevents+json: {e}"))?; + let object = event + .as_object() + .ok_or("cloudevent must be a JSON object")?; + + let id = object + .get("id") + .and_then(Value::as_str) + .ok_or("missing cloudevent id")? + .to_string(); + let name = object + .get("type") + .and_then(Value::as_str) + .ok_or("missing cloudevent type")? + .to_string(); + let content_type = object + .get("datacontenttype") + .and_then(Value::as_str) + .unwrap_or("application/json") + .to_string(); + + // Data: `data` (any JSON; objects/arrays re-serialized to bytes) or + // `data_base64` (binary). + let payload = if let Some(data) = object.get("data") { + match data { + Value::String(s) => s.clone().into_bytes(), + other => serde_json::to_vec(other).map_err(|e| format!("invalid data: {e}"))?, + } + } else if let Some(Value::String(b64)) = object.get("data_base64") { + base64::engine::general_purpose::STANDARD + .decode(b64) + .map_err(|e| format!("invalid data_base64: {e}"))? + } else { + Vec::new() + }; + + // Remaining attributes (source, subject, extensions) become metadata. + let reserved = [ + "specversion", + "id", + "type", + "datacontenttype", + "data", + "data_base64", + ]; + let mut metadata = Vec::new(); + for (key, value) in object { + if reserved.contains(&key.as_str()) { + continue; + } + let value = match value { + Value::String(s) => s.clone(), + other => other.to_string(), + }; + metadata.push((key.clone(), value)); + } + + Ok(Message { + id: Some(id), + name, + kind: MessageKind::Event, + payload, + content_type, + metadata, + }) +} + +/// Render Knative `Trigger` YAML for each event a service subscribes to, derived +/// from its [`SubscriptionPlan`]. Each Trigger filters on the CloudEvent `type` +/// and routes to `subscriber_service` on `broker`. +pub fn knative_triggers(plan: &SubscriptionPlan, broker: &str, subscriber_service: &str) -> String { + let mut out = String::new(); + for event in &plan.events { + let trigger_name = format!("{subscriber_service}-{}", event.replace('.', "-")); + out.push_str(&format!( + "apiVersion: eventing.knative.dev/v1\n\ + kind: Trigger\n\ + metadata:\n\ + \x20 name: {trigger_name}\n\ + spec:\n\ + \x20 broker: {broker}\n\ + \x20 filter:\n\ + \x20 attributes:\n\ + \x20 type: {event}\n\ + \x20 subscriber:\n\ + \x20 ref:\n\ + \x20 apiVersion: serving.knative.dev/v1\n\ + \x20 kind: Service\n\ + \x20 name: {subscriber_service}\n\ + ---\n" + )); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + + fn headers(pairs: &[(&str, &str)]) -> HeaderMap { + let mut map = HeaderMap::new(); + for (k, v) in pairs { + map.insert( + axum::http::HeaderName::from_bytes(k.as_bytes()).unwrap(), + HeaderValue::from_str(v).unwrap(), + ); + } + map + } + + #[test] + fn parses_binary_cloud_event() { + let h = headers(&[ + ("ce-id", "evt-1"), + ("ce-type", "order.created"), + ("ce-source", "/orders"), + ("content-type", "application/json"), + ]); + let body = Bytes::from_static(br#"{"order":"o1"}"#); + let message = parse_cloud_event(&h, &body).unwrap(); + assert_eq!(message.id(), Some("evt-1")); + assert_eq!(message.name(), "order.created"); + assert_eq!(message.payload(), br#"{"order":"o1"}"#); + assert_eq!(message.metadata("source"), Some("/orders")); + } + + #[test] + fn parses_structured_cloud_event() { + let h = headers(&[("content-type", "application/cloudevents+json")]); + let body = Bytes::from( + json!({ + "specversion": "1.0", + "id": "evt-2", + "type": "order.created", + "source": "/orders", + "datacontenttype": "application/json", + "data": {"order": "o2"}, + }) + .to_string(), + ); + let message = parse_cloud_event(&h, &body).unwrap(); + assert_eq!(message.id(), Some("evt-2")); + assert_eq!(message.name(), "order.created"); + assert_eq!(message.payload(), br#"{"order":"o2"}"#); + assert_eq!(message.metadata("source"), Some("/orders")); + } + + #[test] + fn missing_id_is_rejected() { + let h = headers(&[("ce-type", "order.created")]); + assert!(parse_cloud_event(&h, &Bytes::new()).is_err()); + } + + #[test] + fn triggers_render_from_subscription_plan() { + let plan = SubscriptionPlan { + commands: vec![], + events: vec!["seat.reserved".to_string()], + }; + let yaml = knative_triggers(&plan, "default", "checkout-projection"); + assert!(yaml.contains("kind: Trigger")); + assert!(yaml.contains("type: seat.reserved")); + assert!(yaml.contains("name: checkout-projection-seat-reserved")); + assert!(yaml.contains("broker: default")); + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 6da18985a..8a5ae5d14 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -84,6 +84,8 @@ use crate::microsvc::Message; mod capabilities; mod error; mod failure_policy; +#[cfg(feature = "http")] +mod knative; #[cfg(feature = "nats")] mod nats; mod outbox_dispatch; @@ -96,6 +98,8 @@ mod runner; mod source; mod stable_id; +#[cfg(feature = "http")] +pub use knative::{cloud_events_router, knative_triggers}; #[cfg(feature = "nats")] pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; #[cfg(feature = "rabbitmq")] diff --git a/tests/knative_cloudevents/main.rs b/tests/knative_cloudevents/main.rs new file mode 100644 index 000000000..2d2ee2203 --- /dev/null +++ b/tests/knative_cloudevents/main.rs @@ -0,0 +1,141 @@ +//! Knative / CloudEvents HTTP ingress integration tests. +//! +//! Drives `cloud_events_router` over a real ephemeral HTTP server and asserts +//! the CloudEvents binding (binary + structured) and the ack/retry/permanent +//! response-status mapping. Runs in-process — no external broker. +#![cfg(feature = "http")] + +use std::sync::{Arc, Mutex}; + +use serde_json::json; +use sourced_rust::microsvc::transport::cloud_events_router; +use sourced_rust::microsvc::{HandlerError, Service}; + +async fn spawn_server() -> (String, Arc>>) { + let handled = Arc::new(Mutex::new(Vec::::new())); + let h = handled.clone(); + let service = Arc::new( + Service::new(()) + .event("order.created") + .handle(move |ctx| { + h.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({"ok": true})) + }) + .event("flaky") + .handle(|_| { + Err(HandlerError::Repository( + sourced_rust::RepositoryError::Model("transient".into()), + )) + }) + .event("bad") + .handle(|_| Err(HandlerError::Rejected("permanent".into()))), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = cloud_events_router(service); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}/"), handled) +} + +#[tokio::test] +async fn binary_mode_success_returns_200_after_handler() { + let (url, handled) = spawn_server().await; + let client = reqwest::Client::new(); + let resp = client + .post(&url) + .header("ce-id", "evt-1") + .header("ce-type", "order.created") + .header("ce-source", "/orders") + .header("content-type", "application/json") + .body(r#"{"order":"o1"}"#) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(handled.lock().unwrap().clone(), vec!["evt-1".to_string()]); +} + +#[tokio::test] +async fn structured_mode_success_returns_200() { + let (url, handled) = spawn_server().await; + let client = reqwest::Client::new(); + let event = json!({ + "specversion": "1.0", + "id": "evt-2", + "type": "order.created", + "source": "/orders", + "datacontenttype": "application/json", + "data": {"order": "o2"}, + }); + let resp = client + .post(&url) + .header("content-type", "application/cloudevents+json") + .body(event.to_string()) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(handled.lock().unwrap().clone(), vec!["evt-2".to_string()]); +} + +#[tokio::test] +async fn retryable_failure_returns_503() { + let (url, _) = spawn_server().await; + let resp = reqwest::Client::new() + .post(&url) + .header("ce-id", "evt-3") + .header("ce-type", "flaky") + .body("{}") + .send() + .await + .unwrap(); + // Knative should redeliver. + assert_eq!(resp.status(), 503); +} + +#[tokio::test] +async fn permanent_failure_returns_422() { + let (url, _) = spawn_server().await; + let resp = reqwest::Client::new() + .post(&url) + .header("ce-id", "evt-4") + .header("ce-type", "bad") + .body("{}") + .send() + .await + .unwrap(); + // Knative should not retry; its Delivery config dead-letters. + assert_eq!(resp.status(), 422); +} + +#[tokio::test] +async fn unknown_type_returns_422() { + let (url, _) = spawn_server().await; + let resp = reqwest::Client::new() + .post(&url) + .header("ce-id", "evt-5") + .header("ce-type", "no.such.handler") + .body("{}") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422); +} + +#[tokio::test] +async fn missing_id_returns_400() { + let (url, _) = spawn_server().await; + let resp = reqwest::Client::new() + .post(&url) + .header("ce-type", "order.created") + .body("{}") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); +} From 8a78405e972e3ccb23ae63e3409aec4506846e67 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:29:20 -0500 Subject: [PATCH 08/39] feat(transport): Kafka adapter + integration tests + CI KafkaPublisher (acks=all producer ack threshold) and KafkaSource (consumer group, auto-commit off; ack=commit offset, nack=seek-back, dead-letter/park= commit-skip) behind the kafka feature (rdkafka/librdkafka via cmake). recv rides through transient broker-transport errors within a fetch-timeout budget. Adds tests/kafka_transport (verified against apache/kafka:3.8.0 KRaft), a kafka CI job (KRaft service container), and the kafka compose service. Implements [[tasks/kafka-transport-adapter]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/on-pr-quality.yaml | 40 ++++ Cargo.toml | 2 + src/microsvc/transport/kafka.rs | 269 +++++++++++++++++++++++++++ src/microsvc/transport/mod.rs | 4 + tests/kafka_transport/main.rs | 128 +++++++++++++ 5 files changed, 443 insertions(+) create mode 100644 src/microsvc/transport/kafka.rs create mode 100644 tests/kafka_transport/main.rs diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index f10b17213..6bfd18d94 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -95,3 +95,43 @@ jobs: toolchain: stable - name: Run RabbitMQ transport integration tests run: cargo test --test rabbitmq_transport --features rabbitmq --verbose + + kafka: + name: Kafka Integration Tests + runs-on: ubuntu-latest + services: + kafka: + image: apache/kafka:3.8.0 + ports: + - 9092:9092 + env: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + options: >- + --health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092" + --health-interval 10s + --health-timeout 10s + --health-retries 20 + --health-start-period 20s + env: + CARGO_TERM_COLOR: always + KAFKA_BROKERS: localhost:9092 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + # ubuntu-latest ships cmake + gcc, which rdkafka's cmake-build needs. + - name: Run Kafka transport integration tests + run: cargo test --test kafka_transport --features kafka --verbose diff --git a/Cargo.toml b/Cargo.toml index a825f6a84..fb2a226e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ postgres = ["dep:sqlx", "dep:tokio", "sqlx/postgres", "sqlx/runtime-tokio"] sqlite = ["dep:sqlx", "dep:tokio", "sqlx/runtime-tokio", "sqlx/sqlite"] nats = ["dep:async-nats", "dep:futures", "dep:tokio"] rabbitmq = ["dep:lapin", "dep:futures", "dep:tokio"] +kafka = ["dep:rdkafka", "dep:tokio"] [dependencies] async-nats = { version = "0.38", optional = true } @@ -43,6 +44,7 @@ axum = { version = "0.7", optional = true } base64 = "0.22.1" futures = { version = "0.3", optional = true } lapin = { version = "2", optional = true } +rdkafka = { version = "0.36", features = ["cmake-build", "tokio"], optional = true } bitcode = { version = "0.6.9", features = ["serde"] } event-emitter-rs = { version = "0.1.4", optional = true } serde = { version = "1.0.210", features = ["derive"] } diff --git a/src/microsvc/transport/kafka.rs b/src/microsvc/transport/kafka.rs new file mode 100644 index 000000000..39233dc05 --- /dev/null +++ b/src/microsvc/transport/kafka.rs @@ -0,0 +1,269 @@ +//! Kafka transport adapter. +//! +//! [`KafkaPublisher`] sends a canonical [`Message`] to a topic named by the +//! message name, awaiting the producer ack (the durable publish threshold, per +//! the configured `acks`). [`KafkaSource`] consumes with a consumer group +//! (auto-commit disabled) and settles by offset: ack→commit the offset, +//! nack→seek back so the record is re-read, dead-letter/park→commit (skip). +//! +//! Requires the `kafka` feature (builds `librdkafka` via cmake). Integration- +//! tested in `tests/kafka_transport` against a broker (see `compose.yaml`). + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use rdkafka::config::ClientConfig; +use rdkafka::consumer::{Consumer, StreamConsumer}; +use rdkafka::message::{Header, Headers, OwnedHeaders}; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use rdkafka::{Message as KafkaMessageTrait, Offset, TopicPartitionList}; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{AsyncMessagePublisher, TransportError}; +use crate::microsvc::{Message, MessageKind}; + +const MESSAGE_ID_HEADER: &str = "x-sourced-id"; +const MESSAGE_KIND_HEADER: &str = "x-sourced-kind"; + +fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { + TransportError::retryable(format!("{context}: {err}")) +} + +/// Publishes canonical messages to a Kafka topic named by the message name. +pub struct KafkaPublisher { + producer: FutureProducer, + send_timeout: Duration, +} + +impl KafkaPublisher { + /// Wrap an existing producer. + pub fn new(producer: FutureProducer) -> Self { + Self { + producer, + send_timeout: Duration::from_secs(10), + } + } + + /// Connect a producer to `brokers` (comma-separated `host:port`), waiting for + /// `acks=all` so a successful send is durably replicated. + pub async fn connect(brokers: &str) -> Result { + let producer: FutureProducer = ClientConfig::new() + .set("bootstrap.servers", brokers) + .set("acks", "all") + .set("message.timeout.ms", "10000") + .create() + .map_err(|err| retryable("kafka producer", err))?; + Ok(Self::new(producer)) + } +} + +fn owned_headers(message: &Message) -> OwnedHeaders { + let mut headers = OwnedHeaders::new().insert(Header { + key: MESSAGE_KIND_HEADER, + value: Some(kind_str(message.kind)), + }); + if let Some(id) = message.id() { + headers = headers.insert(Header { + key: MESSAGE_ID_HEADER, + value: Some(id), + }); + } + for (key, value) in &message.metadata { + headers = headers.insert(Header { + key: key.as_str(), + value: Some(value.as_str()), + }); + } + headers +} + +impl AsyncMessagePublisher for KafkaPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + let topic = message.name().to_string(); + let key = message.id().unwrap_or(message.name()).to_string(); + let headers = owned_headers(&message); + let record = FutureRecord::to(&topic) + .payload(&message.payload) + .key(&key) + .headers(headers); + self.producer + .send(record, self.send_timeout) + .await + .map_err(|(err, _)| retryable("kafka send", err))?; + Ok(()) + } +} + +/// Consumes a topic with a consumer group, committing offsets on ack. +pub struct KafkaSource { + consumer: Arc, + fetch_timeout: Duration, +} + +impl KafkaSource { + /// Wrap an existing subscribed consumer. + pub fn new(consumer: Arc) -> Self { + Self { + consumer, + fetch_timeout: Duration::from_secs(5), + } + } + + /// How long `recv` waits for a record before returning `Ok(None)`. + pub fn with_fetch_timeout(mut self, timeout: Duration) -> Self { + self.fetch_timeout = timeout; + self + } + + /// Connect a consumer (group `group_id`, auto-commit off, earliest reset) and + /// subscribe to `topics`. + pub async fn connect( + brokers: &str, + group_id: &str, + topics: &[&str], + ) -> Result { + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", brokers) + .set("group.id", group_id) + .set("enable.auto.commit", "false") + .set("auto.offset.reset", "earliest") + .create() + .map_err(|err| retryable("kafka consumer", err))?; + consumer + .subscribe(topics) + .map_err(|err| retryable("kafka subscribe", err))?; + Ok(Self::new(Arc::new(consumer))) + } +} + +impl AsyncMessageSource for KafkaSource { + type Received = KafkaReceived; + + async fn recv(&mut self) -> Result, TransportError> { + // Poll within the fetch-timeout budget. Kafka surfaces transient broker + // transport/coordination errors (normal during group bootstrap and + // rebalances) as recv errors; the client recovers, so we retry until a + // message arrives or the budget elapses (drain → None), rather than + // ending the run on a transient hiccup. + let deadline = Instant::now() + self.fetch_timeout; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + match tokio::time::timeout(remaining, self.consumer.recv()).await { + Ok(Ok(borrowed)) => { + return Ok(Some(KafkaReceived::from_borrowed( + &borrowed, + self.consumer.clone(), + ))); + } + Ok(Err(_transient)) => { + // Back off briefly, then retry within the remaining budget. + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(_elapsed) => return Ok(None), + } + } + } +} + +/// A consumed record plus the means to commit/seek its offset. +pub struct KafkaReceived { + consumer: Arc, + topic: String, + partition: i32, + offset: i64, + message: Message, +} + +impl KafkaReceived { + fn from_borrowed( + borrowed: &rdkafka::message::BorrowedMessage<'_>, + consumer: Arc, + ) -> Self { + let payload = borrowed.payload().map(|p| p.to_vec()).unwrap_or_default(); + let topic = borrowed.topic().to_string(); + let mut id = None; + let mut kind = MessageKind::Event; + let mut metadata = Vec::new(); + if let Some(headers) = borrowed.headers() { + for header in headers.iter() { + let value = header + .value + .map(|v| String::from_utf8_lossy(v).into_owned()) + .unwrap_or_default(); + match header.key { + MESSAGE_ID_HEADER => id = Some(value), + MESSAGE_KIND_HEADER => kind = kind_from_str(&value), + other => metadata.push((other.to_string(), value)), + } + } + } + let mut message = Message::new(topic.clone(), kind, payload); + message.id = id; + message.metadata = metadata; + Self { + consumer, + topic, + partition: borrowed.partition(), + offset: borrowed.offset(), + message, + } + } + + fn commit_offset(&self) -> Result<(), TransportError> { + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&self.topic, self.partition, Offset::Offset(self.offset + 1)) + .map_err(|err| retryable("kafka offset", err))?; + self.consumer + .commit(&tpl, rdkafka::consumer::CommitMode::Sync) + .map_err(|err| retryable("kafka commit", err)) + } +} + +impl ReceivedMessage for KafkaReceived { + fn message(&self) -> &Message { + &self.message + } + + async fn ack(self) -> Result<(), TransportError> { + self.commit_offset() + } + + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + // Do not commit; seek back so this record is re-read (redelivery). + self.consumer + .seek( + &self.topic, + self.partition, + Offset::Offset(self.offset), + Duration::from_secs(5), + ) + .map_err(|err| retryable("kafka seek", err)) + } + + async fn dead_letter(self, _reason: &str) -> Result<(), TransportError> { + // Skip the poison record by committing past it. A DLQ-topic producer is a + // follow-up. + self.commit_offset() + } + + async fn park(self, _reason: &str) -> Result<(), TransportError> { + self.commit_offset() + } +} + +fn kind_str(kind: MessageKind) -> &'static str { + match kind { + MessageKind::Command => "command", + MessageKind::Event => "event", + } +} + +fn kind_from_str(value: &str) -> MessageKind { + match value { + "command" => MessageKind::Command, + _ => MessageKind::Event, + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 8a5ae5d14..d3b25c56c 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -84,6 +84,8 @@ use crate::microsvc::Message; mod capabilities; mod error; mod failure_policy; +#[cfg(feature = "kafka")] +mod kafka; #[cfg(feature = "http")] mod knative; #[cfg(feature = "nats")] @@ -98,6 +100,8 @@ mod runner; mod source; mod stable_id; +#[cfg(feature = "kafka")] +pub use kafka::{KafkaPublisher, KafkaReceived, KafkaSource}; #[cfg(feature = "http")] pub use knative::{cloud_events_router, knative_triggers}; #[cfg(feature = "nats")] diff --git a/tests/kafka_transport/main.rs b/tests/kafka_transport/main.rs new file mode 100644 index 000000000..3e6aecc9a --- /dev/null +++ b/tests/kafka_transport/main.rs @@ -0,0 +1,128 @@ +//! Kafka transport adapter integration tests. +//! +//! Publishes via `KafkaPublisher` (acks=all) and consumes via `KafkaSource` +//! (consumer group, offset commit on ack) against a broker. Skips when +//! `KAFKA_BROKERS` is unset. +#![cfg(feature = "kafka")] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; +use sourced_rust::microsvc::transport::{ + run_source, AsyncMessagePublisher, KafkaPublisher, KafkaSource, RunOptions, +}; +use sourced_rust::microsvc::{Message, MessageKind, Service}; + +static SEQ: AtomicU64 = AtomicU64::new(1); + +fn brokers() -> Option { + match std::env::var("KAFKA_BROKERS") { + Ok(b) => Some(b), + Err(_) => { + eprintln!("skipping kafka transport test: KAFKA_BROKERS is not set"); + None + } + } +} + +fn unique(prefix: &str) -> String { + // Kafka persists topics across runs, so include a per-process time component + // to avoid reading stale messages from a previous run's same-named topic. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("{prefix}-{nanos}-{}", SEQ.fetch_add(1, Ordering::SeqCst)) +} + +#[tokio::test] +async fn publish_then_consume_round_trips_through_kafka() { + let Some(brokers) = brokers() else { return }; + let topic = unique("evt"); + let group = unique("group"); + + // Produce first so the topic auto-creates; then a fresh group reads from + // earliest. + let publisher = KafkaPublisher::connect(&brokers).await.expect("producer"); + for i in 0..3 { + let message = + Message::new(&topic, MessageKind::Event, b"{}".to_vec()).with_id(format!("m{i}")); + publisher.publish(message).await.expect("publish"); + } + + let source = KafkaSource::connect(&brokers, &group, &[&topic]) + .await + .expect("consumer") + .with_fetch_timeout(Duration::from_secs(10)); + + let handled = Arc::new(Mutex::new(Vec::::new())); + let h = handled.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(topic.clone().into_boxed_str())) + .handle(move |ctx| { + h.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + }), + ); + run_source(service, source, RunOptions::idempotent()) + .await + .expect("run_source drains the topic"); + + let mut ids = handled.lock().unwrap().clone(); + ids.sort(); + assert_eq!( + ids, + vec!["m0".to_string(), "m1".to_string(), "m2".to_string()] + ); +} + +#[tokio::test] +async fn message_id_and_metadata_survive_the_round_trip() { + let Some(brokers) = brokers() else { return }; + let topic = unique("evt"); + let group = unique("group"); + + let publisher = KafkaPublisher::connect(&brokers).await.expect("producer"); + publisher + .publish( + Message::new(&topic, MessageKind::Event, br#"{"k":"v"}"#.to_vec()) + .with_id("evt-1") + .with_metadata("correlation_id", "corr-9"), + ) + .await + .expect("publish"); + + let source = KafkaSource::connect(&brokers, &group, &[&topic]) + .await + .expect("consumer") + .with_fetch_timeout(Duration::from_secs(10)); + + let observed = Arc::new(Mutex::new(None)); + let o = observed.clone(); + let service = Arc::new( + Service::new(()) + .event(Box::leak(topic.clone().into_boxed_str())) + .handle(move |ctx| { + let m = ctx.message(); + *o.lock().unwrap() = Some(( + m.id().map(str::to_string), + m.correlation_id().map(str::to_string), + m.payload().to_vec(), + )); + Ok(json!({})) + }), + ); + run_source(service, source, RunOptions::idempotent()) + .await + .unwrap(); + + let got = observed.lock().unwrap().clone().expect("handler ran"); + assert_eq!(got.0.as_deref(), Some("evt-1")); + assert_eq!(got.1.as_deref(), Some("corr-9")); + assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); +} From 2b1b372557083de854395a9b63741a82f895d8a1 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:35:23 -0500 Subject: [PATCH 09/39] ci: extract integration tests to reusable workflows; run on push-to-main Moves the postgres/nats/rabbitmq/kafka integration jobs into reusable workflow_call files (.github/workflows/integration-*.yaml) referenced via local ./ paths from both on-pr-quality and on-push-main-version-and-tag. The push-to- main pipeline now runs all broker integration tests and gates version-and-tag on them. Validated with actionlint. Relates to [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/integration-kafka.yaml | 47 +++++++ .github/workflows/integration-nats.yaml | 32 +++++ .github/workflows/integration-postgres.yaml | 41 ++++++ .github/workflows/integration-rabbitmq.yaml | 33 +++++ .github/workflows/on-pr-quality.yaml | 121 +----------------- .../on-push-main-version-and-tag.yaml | 43 ++----- 6 files changed, 168 insertions(+), 149 deletions(-) create mode 100644 .github/workflows/integration-kafka.yaml create mode 100644 .github/workflows/integration-nats.yaml create mode 100644 .github/workflows/integration-postgres.yaml create mode 100644 .github/workflows/integration-rabbitmq.yaml diff --git a/.github/workflows/integration-kafka.yaml b/.github/workflows/integration-kafka.yaml new file mode 100644 index 000000000..a2c05efbd --- /dev/null +++ b/.github/workflows/integration-kafka.yaml @@ -0,0 +1,47 @@ +name: Kafka Integration Tests + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-kafka.yaml` +# from both the PR-quality and push-to-main pipelines. +on: + workflow_call: + +jobs: + kafka: + name: Kafka Integration Tests + runs-on: ubuntu-latest + services: + kafka: + image: apache/kafka:3.8.0 + ports: + - 9092:9092 + env: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + options: >- + --health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092" + --health-interval 10s + --health-timeout 10s + --health-retries 20 + --health-start-period 20s + env: + CARGO_TERM_COLOR: always + KAFKA_BROKERS: localhost:9092 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + # ubuntu-latest ships cmake + gcc, which rdkafka's cmake-build needs. + - name: Run Kafka transport integration tests + run: cargo test --test kafka_transport --features kafka --verbose diff --git a/.github/workflows/integration-nats.yaml b/.github/workflows/integration-nats.yaml new file mode 100644 index 000000000..7385a4b92 --- /dev/null +++ b/.github/workflows/integration-nats.yaml @@ -0,0 +1,32 @@ +name: NATS JetStream Integration Tests + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-nats.yaml` +# from both the PR-quality and push-to-main pipelines. +on: + workflow_call: + +jobs: + nats: + name: NATS JetStream Integration Tests + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + NATS_URL: nats://localhost:4222 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + # NATS service containers can't override the entrypoint to enable + # JetStream, so start it as a step instead. + - name: Start NATS (JetStream) + run: | + docker run -d --name nats -p 4222:4222 nats:2.10-alpine -js + for i in $(seq 1 30); do + if nc -z localhost 4222; then echo "nats up"; break; fi + sleep 1 + done + - name: Run NATS transport integration tests + run: cargo test --test nats_transport --features nats --verbose diff --git a/.github/workflows/integration-postgres.yaml b/.github/workflows/integration-postgres.yaml new file mode 100644 index 000000000..63bc0d7e6 --- /dev/null +++ b/.github/workflows/integration-postgres.yaml @@ -0,0 +1,41 @@ +name: Postgres Integration Tests + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-postgres.yaml` +# from both the PR-quality and push-to-main pipelines. +on: + workflow_call: + +jobs: + postgres: + name: Postgres Integration Tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sourced_rust_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d sourced_rust_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + CARGO_TERM_COLOR: always + DATABASE_URL: postgres://postgres:postgres@localhost:5432/sourced_rust_test + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + - name: Run Postgres integration tests + run: | + cargo test --test postgres_repository --all-features --verbose + cargo test --test postgres_repository_conformance --all-features --verbose + cargo test --test postgres_transport --all-features --verbose + cargo test --test distributed_read_model --all-features --verbose diff --git a/.github/workflows/integration-rabbitmq.yaml b/.github/workflows/integration-rabbitmq.yaml new file mode 100644 index 000000000..4731e6bcf --- /dev/null +++ b/.github/workflows/integration-rabbitmq.yaml @@ -0,0 +1,33 @@ +name: RabbitMQ Integration Tests + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-rabbitmq.yaml` +# from both the PR-quality and push-to-main pipelines. +on: + workflow_call: + +jobs: + rabbitmq: + name: RabbitMQ Integration Tests + runs-on: ubuntu-latest + services: + rabbitmq: + image: rabbitmq:3.13-management-alpine + ports: + - 5672:5672 + options: >- + --health-cmd "rabbitmq-diagnostics -q ping" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + CARGO_TERM_COLOR: always + AMQP_URL: amqp://guest:guest@localhost:5672/%2f + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + - name: Run RabbitMQ transport integration tests + run: cargo test --test rabbitmq_transport --features rabbitmq --verbose diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index 6bfd18d94..1cdba6b67 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -12,126 +12,13 @@ jobs: cargo_incremental: true postgres: - name: Postgres Integration Tests - runs-on: ubuntu-latest - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: sourced_rust_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d sourced_rust_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - env: - CARGO_TERM_COLOR: always - DATABASE_URL: postgres://postgres:postgres@localhost:5432/sourced_rust_test - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 - with: - toolchain: stable - - name: Run Postgres integration tests - run: | - cargo test --test postgres_repository --all-features --verbose - cargo test --test postgres_repository_conformance --all-features --verbose - cargo test --test postgres_transport --all-features --verbose - cargo test --test distributed_read_model --all-features --verbose + uses: ./.github/workflows/integration-postgres.yaml nats: - name: NATS JetStream Integration Tests - runs-on: ubuntu-latest - env: - CARGO_TERM_COLOR: always - NATS_URL: nats://localhost:4222 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 - with: - toolchain: stable - # NATS service containers can't override the entrypoint to enable - # JetStream, so start it as a step instead. - - name: Start NATS (JetStream) - run: | - docker run -d --name nats -p 4222:4222 nats:2.10-alpine -js - for i in $(seq 1 30); do - if nc -z localhost 4222; then echo "nats up"; break; fi - sleep 1 - done - - name: Run NATS transport integration tests - run: cargo test --test nats_transport --features nats --verbose + uses: ./.github/workflows/integration-nats.yaml rabbitmq: - name: RabbitMQ Integration Tests - runs-on: ubuntu-latest - services: - rabbitmq: - image: rabbitmq:3.13-management-alpine - ports: - - 5672:5672 - options: >- - --health-cmd "rabbitmq-diagnostics -q ping" - --health-interval 5s - --health-timeout 5s - --health-retries 20 - env: - CARGO_TERM_COLOR: always - AMQP_URL: amqp://guest:guest@localhost:5672/%2f - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 - with: - toolchain: stable - - name: Run RabbitMQ transport integration tests - run: cargo test --test rabbitmq_transport --features rabbitmq --verbose + uses: ./.github/workflows/integration-rabbitmq.yaml kafka: - name: Kafka Integration Tests - runs-on: ubuntu-latest - services: - kafka: - image: apache/kafka:3.8.0 - ports: - - 9092:9092 - env: - KAFKA_NODE_ID: 1 - KAFKA_PROCESS_ROLES: broker,controller - KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - options: >- - --health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092" - --health-interval 10s - --health-timeout 10s - --health-retries 20 - --health-start-period 20s - env: - CARGO_TERM_COLOR: always - KAFKA_BROKERS: localhost:9092 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 - with: - toolchain: stable - # ubuntu-latest ships cmake + gcc, which rdkafka's cmake-build needs. - - name: Run Kafka transport integration tests - run: cargo test --test kafka_transport --features kafka --verbose + uses: ./.github/workflows/integration-kafka.yaml diff --git a/.github/workflows/on-push-main-version-and-tag.yaml b/.github/workflows/on-push-main-version-and-tag.yaml index 5e81ee898..cb52d68c0 100644 --- a/.github/workflows/on-push-main-version-and-tag.yaml +++ b/.github/workflows/on-push-main-version-and-tag.yaml @@ -18,43 +18,22 @@ jobs: cargo_incremental: true postgres: - name: Postgres Integration Tests - runs-on: ubuntu-latest - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: sourced_rust_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d sourced_rust_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - env: - CARGO_TERM_COLOR: always - DATABASE_URL: postgres://postgres:postgres@localhost:5432/sourced_rust_test - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 - with: - toolchain: stable - - name: Run Postgres integration tests - run: | - cargo test --test postgres_repository --all-features --verbose - cargo test --test postgres_repository_conformance --all-features --verbose - cargo test --test distributed_read_model --all-features --verbose + uses: ./.github/workflows/integration-postgres.yaml + + nats: + uses: ./.github/workflows/integration-nats.yaml + + rabbitmq: + uses: ./.github/workflows/integration-rabbitmq.yaml + + kafka: + uses: ./.github/workflows/integration-kafka.yaml # This uses commit logs and tags from git to determine the next version number and create a tag for the release. # Some commits such as chore: will not trigger a version bump and tag; this is by design. version-and-tag: name: Version and Tag - needs: [quality, postgres] + needs: [quality, postgres, nats, rabbitmq, kafka] uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.20.2 secrets: DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} From efbf5bcad297a8688f1fdb5864b25e95f318bc9b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 22:37:19 -0500 Subject: [PATCH 10/39] docs(transport): add async transports guide docs/async-transports.md documents the transport layer: core contracts, the two confirmation thresholds, the source runner, the publisher/outbox dispatcher, all five adapters (in-memory, Postgres, NATS, RabbitMQ, Kafka, Knative), and how to run the conformance + broker integration tests. Progresses [[tasks/transport-docs-examples-cutover]] under [[tasks/async-transport-implementation]]. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/async-transports.md | 124 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docs/async-transports.md diff --git a/docs/async-transports.md b/docs/async-transports.md new file mode 100644 index 000000000..00a612910 --- /dev/null +++ b/docs/async-transports.md @@ -0,0 +1,124 @@ +# Async Microservice Transports + +Distributed (published from the `sourced_rust` crate) keeps the synchronous +in-memory bus intact and adds an async transport layer under +`microsvc::transport`. The design line is: + +- **`microsvc`** owns handler registration, guards, typed input decoding, + dispatch, and handler metadata; +- **transport adapters** own how messages are received, acknowledged, retried, + published, and mapped to external topics/subjects/queues/routes. + +The shared vocabulary lives in `microsvc::transport` and depends on no concrete +broker. The same application code runs over any transport — selecting one is an +adapter/wiring change, not a handler change. + +## Core vocabulary + +| Type | Purpose | +| --- | --- | +| `TransportError` / `TransportErrorKind` | Retryable vs permanent classification. Drives redelivery vs the failure policy. | +| `FailurePolicy` / `FailureAction` | What happens to a permanent failure: `Retry`, `DeadLetter`, `Park`, `LogAndAck`, `Stop`. | +| `RunOptions` / `ConsumerDeliveryMode` / `InboxHook` | Idempotent dispatch by default; placeholder hook for the future consumer inbox. | +| `TransportCapabilities` | Per-transport receive durability, publish confirmation, retry ownership, ack kind, Knative integration. | +| `validate_stable_message_id` | Rules an inbox-enabled run uses to reject messages lacking a usable dedup key. | + +### Two confirmation thresholds + +Producing and consuming have *separate* completion thresholds: + +- **Producer publish threshold** — when an outbox row may be marked published: + Postgres transaction commit, RabbitMQ publisher confirm, Kafka producer ack + (`acks`), NATS JetStream publish ack, Knative 2xx, in-memory acceptance. Only + then is the row complete; an unknown outcome stays retryable. +- **Consumer ack threshold** — when the adapter may acknowledge receipt: only + after the handler (and any inbox receipt) committed. The default never + silently acks a handler error — retryable failures redeliver, permanent + failures go through the `FailurePolicy`. + +## Receiving: `AsyncMessageSource` + `run_source` + +Direct transports implement `AsyncMessageSource` (pull a message) and +`ReceivedMessage` (settle it). `run_source` drives the loop, dispatching through +`Service::dispatch_message` and settling only after the handler completes: + +```rust,ignore +use sourced_rust::microsvc::transport::{run_source, RunOptions}; + +run_source(service, source, RunOptions::idempotent()).await?; +``` + +The runner acks on success, nacks retryable failures for redelivery, routes +permanent failures through the failure policy, **acks-and-ignores** messages with +no registered handler (so fan-out transports can over-deliver), stops gracefully +when the source drains, and never swallows receive/settle errors. Inbox mode +(`RunOptions::inbox(hook)`) enforces a stable message id before dispatch. + +## Publishing: `AsyncMessagePublisher` + outbox + +`AsyncMessagePublisher` is the single publish boundary; each adapter documents +its publish threshold. `OutboxDispatcher` bridges durable outbox rows to a +publisher, sharing one claim → publish → complete path between background polling +(`dispatch_batch`) and after-commit immediate dispatch (`dispatch_ids`): + +```rust,ignore +let dispatcher = OutboxDispatcher::new(store, publisher, "worker-1", lease, max_attempts); +let outcome = dispatcher.dispatch_ids(&committed_ids).await?; // claim-before-publish +``` + +A row completes only after `publish()` resolves `Ok`; an unknown/failed publish +leaves it retryable (release until the attempt ceiling, then fail). Outbox rows +map to a canonical `Message` via `From<&OutboxMessage>`; framework-derived +metadata (codec, destination, source aggregate) is namespaced under the reserved +`x-sourced-` prefix so it cannot be shadowed by user metadata. + +## Adapters + +| Transport | Feature | Source / Publisher | Notes | +| --- | --- | --- | --- | +| In-memory | (always) | conformance fakes | Reference adapter; reused by `transport_conformance`. | +| Postgres | (always) | `OutboxSource` | Outbox-backed durable receive: `FOR UPDATE SKIP LOCKED` + lease, ack→complete, nack→release, dead-letter/park→fail. The starter durable transport. | +| NATS JetStream | `nats` | `NatsJetStreamSource` / `NatsPublisher` | ack/nak/term; stable id rides as `Nats-Msg-Id` (also the dedup key). | +| RabbitMQ | `rabbitmq` | `RabbitSource` / `RabbitPublisher` | Publisher confirms; `basic_get`; ack/nack-requeue/reject. | +| Kafka | `kafka` | `KafkaSource` / `KafkaPublisher` | `acks=all`; consumer-group offset commit on ack, seek-back on nack. | +| Knative / HTTP | `http` | `cloud_events_router` (ingress) | Endpoint-driven, not a polling source; 200 success / 503 retryable / 422 permanent; `knative_triggers()` renders Trigger YAML from `subscription_plan()`. | + +Postgres is the low-ops starter: one Postgres cluster can back repositories, +read models, outbox, and durable transport. (`sqlxmq` was evaluated but its +push-based `JobRegistry` does not fit the pull-based `AsyncMessageSource` / +`run_source` boundary, so the proven durable-queue patterns were borrowed rather +than the crate — see `tasks/postgres-transport-adapter-first-pass`.) + +Retry/backoff/dead-lettering ownership differs: with Knative it is +**platform-managed** (Delivery/Trigger config); with direct transports the +adapter and this crate own it via the `FailurePolicy` and the outbox lease. + +## Testing + +The reusable conformance harness (`tests/transport_conformance/`) proves the +contract with adapter-neutral fakes; `tests/transport_in_memory/` runs it as the +in-memory reference. Real-broker integration tests are feature-gated and skip +when their env var is unset: + +```sh +docker compose up -d # postgres, rabbitmq, kafka, nats (see compose.yaml) + +DATABASE_URL=postgres://sourced:sourced@localhost:5432/sourced_rust \ + cargo test --test postgres_transport --features postgres +NATS_URL=nats://localhost:4222 cargo test --test nats_transport --features nats +AMQP_URL=amqp://guest:guest@localhost:5672/%2f \ + cargo test --test rabbitmq_transport --features rabbitmq +KAFKA_BROKERS=localhost:9092 cargo test --test kafka_transport --features kafka +``` + +Each broker has a matching GitHub Actions job (reusable +`.github/workflows/integration-*.yaml`) that runs on PRs and on push to `main`. + +## Status + +Implemented and verified: the core contracts, the source runner, the publisher / +outbox dispatcher, the conformance harness, the Postgres / NATS / RabbitMQ / +Kafka adapters, and the Knative ingress. Still open: migrating the in-repo +examples to showcase these APIs and removing the legacy synchronous bus paths +(a breaking change), and a long-running poll/notify consumer daemon for the +Postgres source. See `tasks/transport-docs-examples-cutover`. From 5bab4ec97e0903db942139c496731a0d25e2ef40 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 23:48:02 -0500 Subject: [PATCH 11/39] feat(transport): add Bus/BusConsumer facade + InMemoryBus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the ergonomic servicebus-style surface over the async transport traits: `Bus` (produce — send/publish/send_message/ publish_message) and `BusConsumer` (consume — listen/subscribe, generic over the service data, deriving message names from the service's command/event handlers and running through run_source). Knative will implement only `Bus` (it consumes via generated Triggers + the HTTP ingress); pull transports implement both. InMemoryBus is the dev/test reference implementation: competing- consumer queues back send/listen (point-to-point, each message popped once) and retained per-subscriber-cursor logs back publish/subscribe (fan-out — every subscriber sees every event), the in-memory shape of the Postgres-as-log fan-out model. 5 unit tests cover both semantics plus unknown-command-ignored and handler-error-via-failure-policy. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/bus.rs | 79 ++++++ src/microsvc/transport/in_memory_bus.rs | 330 ++++++++++++++++++++++++ src/microsvc/transport/mod.rs | 4 + 3 files changed, 413 insertions(+) create mode 100644 src/microsvc/transport/bus.rs create mode 100644 src/microsvc/transport/in_memory_bus.rs diff --git a/src/microsvc/transport/bus.rs b/src/microsvc/transport/bus.rs new file mode 100644 index 000000000..ffe9b1a94 --- /dev/null +++ b/src/microsvc/transport/bus.rs @@ -0,0 +1,79 @@ +//! The ergonomic bus surface: `send`/`listen` (point-to-point commands) and +//! `publish`/`subscribe` (fan-out events), mirroring the Node `servicebus` +//! family (`rabbitbus`/`kafkabus`/`knativebus`). +//! +//! The surface is split so producing (uniform across every transport) and +//! consuming (a `run_source` loop for pull transports, but generated manifests +//! for Knative) stay coherent: +//! +//! - [`Bus`] — produce: `send` a command, `publish` an event. Every transport. +//! - [`BusConsumer`] — consume: `listen` for commands (competing), `subscribe` +//! to events (fan-out). Pull transports only (in-memory, NATS, RabbitMQ, +//! Kafka, Postgres). Knative consumes via generated Triggers + the HTTP +//! ingress, so it implements only [`Bus`]. +//! +//! A concrete `*Bus` implements both, so `bus.send/listen/publish/subscribe` all +//! work on it. `send`/`publish` lower to the transport's [`AsyncMessagePublisher`]; +//! `listen`/`subscribe` build the transport's [`AsyncMessageSource`] with the +//! right topology and run it through the shared [`run_source`](super::run_source). + +use std::future::Future; +use std::sync::Arc; + +use super::{Message, RunOptions, TransportError}; +use crate::microsvc::Service; + +/// Produce side of the bus — uniform across every transport. +pub trait Bus: Send + Sync { + /// Send a point-to-point command (1:1, competing consumers). + fn send( + &self, + name: &str, + payload: Vec, + ) -> impl Future> + Send; + + /// Publish a fan-out event (1:N). + fn publish( + &self, + name: &str, + payload: Vec, + ) -> impl Future> + Send; + + /// Send a fully-formed command message (explicit id/metadata/content-type). + fn send_message( + &self, + message: Message, + ) -> impl Future> + Send; + + /// Publish a fully-formed event message. + fn publish_message( + &self, + message: Message, + ) -> impl Future> + Send; +} + +/// Consume side of the bus — pull transports that run a [`run_source`] loop. +/// +/// `listen`/`subscribe` derive the message names from the service's registered +/// handlers ([`Service::command_names`]/[`Service::event_names`]), build the +/// transport's source with the matching topology, and run it. Both run until the +/// source drains/stops. +/// +/// [`run_source`]: super::run_source +pub trait BusConsumer: Send + Sync { + /// Run `service` as a command listener: consume its command names with + /// competing-consumer (point-to-point) semantics. + fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> impl Future> + Send; + + /// Run `service` as an event subscriber: consume its event names with + /// fan-out semantics. + fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> impl Future> + Send; +} diff --git a/src/microsvc/transport/in_memory_bus.rs b/src/microsvc/transport/in_memory_bus.rs new file mode 100644 index 000000000..01929a79e --- /dev/null +++ b/src/microsvc/transport/in_memory_bus.rs @@ -0,0 +1,330 @@ +//! In-memory bus — the dev/test reference implementation of [`Bus`] + +//! [`BusConsumer`]. +//! +//! `send`/`listen` use named queues with competing-consumer (point-to-point) +//! semantics: a message is popped by exactly one consumer. `publish`/`subscribe` +//! use named **retained logs** with a per-subscriber cursor, so every subscriber +//! sees every event (fan-out) — the same log+offset shape the Postgres fan-out +//! transport uses, in memory. +//! +//! It is intentionally simple (no durability, no redelivery on nack) — for tests +//! and local development. Use a real transport for production reliability. + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex}; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{run_source, Bus, BusConsumer, RunOptions, TransportError}; +use crate::microsvc::{Message, MessageKind, Service}; + +type Queues = Arc>>>; +type Topics = Arc>>>; + +fn lock_poisoned(what: &str) -> TransportError { + TransportError::permanent(format!("in-memory bus {what} lock poisoned")) +} + +/// In-memory [`Bus`] + [`BusConsumer`] for tests and local development. +/// +/// Cheap to clone (shares the same queues/logs), so competing listeners and +/// fan-out subscribers can each hold a clone. +#[derive(Clone, Default)] +pub struct InMemoryBus { + queues: Queues, + topics: Topics, +} + +impl InMemoryBus { + pub fn new() -> Self { + Self::default() + } + + fn enqueue(&self, message: Message) -> Result<(), TransportError> { + self.queues + .lock() + .map_err(|_| lock_poisoned("queue"))? + .entry(message.name().to_string()) + .or_default() + .push_back(message); + Ok(()) + } + + fn append(&self, message: Message) -> Result<(), TransportError> { + self.topics + .lock() + .map_err(|_| lock_poisoned("topic"))? + .entry(message.name().to_string()) + .or_default() + .push(message); + Ok(()) + } +} + +impl Bus for InMemoryBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.enqueue(Message::new(name, MessageKind::Command, payload)) + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.append(Message::new(name, MessageKind::Event, payload)) + } + + async fn send_message(&self, message: Message) -> Result<(), TransportError> { + self.enqueue(message) + } + + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.append(message) + } +} + +impl BusConsumer for InMemoryBus { + async fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let names = service + .command_names() + .iter() + .map(|n| n.to_string()) + .collect(); + let source = QueueSource { + queues: self.queues.clone(), + names, + }; + run_source(service, source, options).await + } + + async fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let names = service + .event_names() + .iter() + .map(|n| n.to_string()) + .collect(); + let source = TopicSource { + topics: self.topics.clone(), + names, + cursors: HashMap::new(), + }; + run_source(service, source, options).await + } +} + +/// Competing-consumer source over the named queues: each message is popped once. +struct QueueSource { + queues: Queues, + names: Vec, +} + +impl AsyncMessageSource for QueueSource { + type Received = InMemoryReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let mut queues = self.queues.lock().map_err(|_| lock_poisoned("queue"))?; + for name in &self.names { + if let Some(message) = queues.get_mut(name).and_then(VecDeque::pop_front) { + return Ok(Some(InMemoryReceived { message })); + } + } + Ok(None) + } +} + +/// Fan-out source over the named retained logs: each `TopicSource` has its own +/// cursor, so every subscriber reads every event. +struct TopicSource { + topics: Topics, + names: Vec, + cursors: HashMap, +} + +impl AsyncMessageSource for TopicSource { + type Received = InMemoryReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let topics = self.topics.lock().map_err(|_| lock_poisoned("topic"))?; + for name in &self.names { + let Some(log) = topics.get(name) else { + continue; + }; + let cursor = self.cursors.entry(name.clone()).or_insert(0); + if *cursor < log.len() { + let message = log[*cursor].clone(); + *cursor += 1; + return Ok(Some(InMemoryReceived { message })); + } + } + Ok(None) + } +} + +/// In-memory delivery. Settling is a no-op: queue pops and log cursors already +/// advanced on `recv`, and the in-memory bus does not redeliver. +pub struct InMemoryReceived { + message: Message, +} + +impl ReceivedMessage for InMemoryReceived { + fn message(&self) -> &Message { + &self.message + } + async fn ack(self) -> Result<(), TransportError> { + Ok(()) + } + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::HandlerError; + use serde_json::json; + use std::future::Future; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + fn recorder() -> Arc>> { + Arc::new(Mutex::new(Vec::new())) + } + + fn command_service(rec: Arc>>) -> Arc> { + Arc::new(Service::new(()).command("work").handle(move |ctx| { + rec.lock().unwrap().push(ctx.message().name().to_string()); + Ok(json!({})) + })) + } + + fn event_service(rec: Arc>>) -> Arc> { + Arc::new(Service::new(()).event("evt").handle(move |ctx| { + rec.lock() + .unwrap() + .push(ctx.message().id().unwrap_or("?").to_string()); + Ok(json!({})) + })) + } + + #[test] + fn send_then_listen_dispatches_each_command() { + let bus = InMemoryBus::new(); + for _ in 0..3 { + block_on(bus.send("work", b"{}".to_vec())).unwrap(); + } + let rec = recorder(); + block_on(bus.listen(command_service(rec.clone()), RunOptions::idempotent())).unwrap(); + assert_eq!( + rec.lock().unwrap().len(), + 3, + "the listener handles all 3 commands" + ); + } + + #[test] + fn listen_is_point_to_point_each_message_popped_once() { + // Two competing sources over the same queue: each message goes to one. + let bus = InMemoryBus::new(); + for i in 0..4 { + block_on(bus.send_message( + Message::new("work", MessageKind::Command, b"{}".to_vec()).with_id(format!("m{i}")), + )) + .unwrap(); + } + let mut a = QueueSource { + queues: bus.queues.clone(), + names: vec!["work".to_string()], + }; + let mut b = QueueSource { + queues: bus.queues.clone(), + names: vec!["work".to_string()], + }; + let mut got = Vec::new(); + // Alternate; each pop removes the message (competing). + for _ in 0..4 { + if let Some(r) = block_on(a.recv()).unwrap() { + got.push(r.message().id().unwrap().to_string()); + } + if let Some(r) = block_on(b.recv()).unwrap() { + got.push(r.message().id().unwrap().to_string()); + } + } + got.sort(); + assert_eq!( + got, + vec!["m0", "m1", "m2", "m3"], + "each message delivered exactly once" + ); + // Queue now drained for both. + assert!(block_on(a.recv()).unwrap().is_none()); + assert!(block_on(b.recv()).unwrap().is_none()); + } + + #[test] + fn publish_then_subscribe_fans_out_to_every_subscriber() { + let bus = InMemoryBus::new(); + for i in 0..3 { + block_on(bus.publish_message( + Message::new("evt", MessageKind::Event, b"{}".to_vec()).with_id(format!("e{i}")), + )) + .unwrap(); + } + // Two independent subscribers; each gets every event (own cursor). + let a = recorder(); + let b = recorder(); + block_on(bus.subscribe(event_service(a.clone()), RunOptions::idempotent())).unwrap(); + block_on(bus.subscribe(event_service(b.clone()), RunOptions::idempotent())).unwrap(); + let mut a_ids = a.lock().unwrap().clone(); + let mut b_ids = b.lock().unwrap().clone(); + a_ids.sort(); + b_ids.sort(); + assert_eq!(a_ids, vec!["e0", "e1", "e2"]); + assert_eq!(b_ids, vec!["e0", "e1", "e2"]); + } + + #[test] + fn unknown_command_is_acked_and_ignored() { + // A command with no handler is ignored by the runner (acked), not an error. + let bus = InMemoryBus::new(); + block_on(bus.send("unrelated", b"{}".to_vec())).unwrap(); + block_on(bus.send("work", b"{}".to_vec())).unwrap(); + let rec = recorder(); + block_on(bus.listen(command_service(rec.clone()), RunOptions::idempotent())).unwrap(); + assert_eq!(rec.lock().unwrap().clone(), vec!["work"]); + } + + #[test] + fn handler_error_does_not_panic_the_loop() { + let bus = InMemoryBus::new(); + block_on(bus.send("work", b"{}".to_vec())).unwrap(); + let service: Arc> = Arc::new( + Service::new(()) + .command("work") + .handle(|_| Err(HandlerError::Rejected("no".into()))), + ); + // Default failure policy dead-letters the permanent failure; in-memory + // dead_letter is a no-op nack, so the run completes cleanly. + block_on(bus.listen(service, RunOptions::idempotent())).unwrap(); + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index d3b25c56c..cc3c44682 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -81,9 +81,11 @@ use crate::microsvc::Message; +mod bus; mod capabilities; mod error; mod failure_policy; +mod in_memory_bus; #[cfg(feature = "kafka")] mod kafka; #[cfg(feature = "http")] @@ -109,9 +111,11 @@ pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; #[cfg(feature = "rabbitmq")] pub use rabbitmq::{RabbitPublisher, RabbitReceived, RabbitSource}; +pub use bus::{Bus, BusConsumer}; pub use capabilities::{ConsumerAckKind, KnativeIntegrationKind, TransportCapabilities}; pub use error::{TransportError, TransportErrorKind}; pub use failure_policy::{FailureAction, FailurePolicy}; +pub use in_memory_bus::{InMemoryBus, InMemoryReceived}; pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; pub use outbox_source::{ OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, From d59ec24536af7de0debdbf14c3621cac6a8c9a82 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 28 May 2026 23:56:35 -0500 Subject: [PATCH 12/39] feat(transport): add NatsBus (send/listen + publish/subscribe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NatsBus implements Bus + BusConsumer over one JetStream stream bound to `{namespace}.>`: - send → subject `{ns}.cmd.{name}` (command) - publish → subject `{ns}.evt.{name}` (event) - listen → durable pull consumer `{group}_cmd` filtered to the service's command subjects; replicas sharing a group share the durable, so JetStream load-balances — point-to-point / competing-consumer. - subscribe → durable `{group}_evt`; each distinct group gets its own durable on the shared stream, so every group sees every event — fan-out. Adds NatsJetStreamSource::with_strip_prefix (default off, backwards compatible) so the dispatched message name is the bare name once the `{ns}.cmd.`/`{ns}.evt.` subject prefix is removed. Two integration tests prove competing-across-a-group (each command handled exactly once by concurrent replicas) and fan-out-across-groups (every group sees every event) against a live JetStream server. Also makes tests/nats_transport unique() process-unique so re-runs against a persistent server don't collide with leftover stream/consumer state. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/mod.rs | 4 + src/microsvc/transport/nats.rs | 29 +++- src/microsvc/transport/nats_bus.rs | 214 +++++++++++++++++++++++++++++ tests/nats_transport/main.rs | 136 +++++++++++++++++- 4 files changed, 377 insertions(+), 6 deletions(-) create mode 100644 src/microsvc/transport/nats_bus.rs diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index cc3c44682..83528cb16 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -92,6 +92,8 @@ mod kafka; mod knative; #[cfg(feature = "nats")] mod nats; +#[cfg(feature = "nats")] +mod nats_bus; mod outbox_dispatch; mod outbox_source; mod publisher; @@ -108,6 +110,8 @@ pub use kafka::{KafkaPublisher, KafkaReceived, KafkaSource}; pub use knative::{cloud_events_router, knative_triggers}; #[cfg(feature = "nats")] pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; +#[cfg(feature = "nats")] +pub use nats_bus::NatsBus; #[cfg(feature = "rabbitmq")] pub use rabbitmq::{RabbitPublisher, RabbitReceived, RabbitSource}; diff --git a/src/microsvc/transport/nats.rs b/src/microsvc/transport/nats.rs index 3343bc22e..f98199620 100644 --- a/src/microsvc/transport/nats.rs +++ b/src/microsvc/transport/nats.rs @@ -102,6 +102,7 @@ impl AsyncMessagePublisher for NatsPublisher { pub struct NatsJetStreamSource { consumer: Consumer, fetch_timeout: Duration, + strip_prefix: Option, } impl NatsJetStreamSource { @@ -110,6 +111,7 @@ impl NatsJetStreamSource { Self { consumer, fetch_timeout: Duration::from_millis(500), + strip_prefix: None, } } @@ -119,6 +121,17 @@ impl NatsJetStreamSource { self } + /// Strip `prefix` from each delivered subject when deriving the message name, + /// so a subject like `app.cmd.account.debit` becomes the name `account.debit`. + /// + /// Used by [`NatsBus`](super::NatsBus), which namespaces commands and events + /// under `{ns}.cmd.` / `{ns}.evt.` subjects. Default: no stripping (the full + /// subject is the name). + pub fn with_strip_prefix(mut self, prefix: impl Into) -> Self { + self.strip_prefix = Some(prefix.into()); + self + } + /// Connect to a NATS server URL, then create/open the stream + consumer. pub async fn connect( url: &str, @@ -179,7 +192,10 @@ impl AsyncMessageSource for NatsJetStreamSource { .map_err(|err| retryable("nats fetch", err))?; match batch.next().await { - Some(Ok(message)) => Ok(Some(NatsReceived::from_jetstream(message))), + Some(Ok(message)) => Ok(Some(NatsReceived::from_jetstream( + message, + self.strip_prefix.as_deref(), + ))), Some(Err(err)) => Err(retryable("nats batch message", err)), None => Ok(None), } @@ -193,8 +209,15 @@ pub struct NatsReceived { } impl NatsReceived { - fn from_jetstream(raw: jetstream::Message) -> Self { - let name = raw.subject.to_string(); + fn from_jetstream(raw: jetstream::Message, strip_prefix: Option<&str>) -> Self { + let subject = raw.subject.to_string(); + let name = match strip_prefix { + Some(prefix) => subject + .strip_prefix(prefix) + .map(str::to_string) + .unwrap_or(subject), + None => subject, + }; let payload = raw.payload.to_vec(); let mut id = None; let mut kind = MessageKind::Event; diff --git a/src/microsvc/transport/nats_bus.rs b/src/microsvc/transport/nats_bus.rs new file mode 100644 index 000000000..487008385 --- /dev/null +++ b/src/microsvc/transport/nats_bus.rs @@ -0,0 +1,214 @@ +//! NATS JetStream [`Bus`] + [`BusConsumer`]. +//! +//! One JetStream stream backs the bus, bound to `{namespace}.>`. The two bus +//! surfaces map onto subjects and durable consumers like this: +//! +//! - `send(name)` → subject `{namespace}.cmd.{name}` (command). +//! - `publish(name)` → subject `{namespace}.evt.{name}` (event). +//! - `listen` → a durable pull consumer named `{group}.cmd`, filtered to the +//! service's command subjects. All replicas sharing a `group` bind the **same** +//! durable, so JetStream load-balances commands across them — point-to-point / +//! competing-consumer. +//! - `subscribe` → a durable pull consumer named `{group}.evt`, filtered to the +//! service's event subjects. Each distinct `group` gets its **own** durable on +//! the shared stream, so every group sees every event — fan-out. +//! +//! The `group` is the logical consumer identity (the service/deployment name). +//! Same group ⇒ competing; different groups ⇒ independent fan-out copies. +//! +//! Requires the `nats` feature. Integration-tested in `tests/nats_transport`. + +use std::sync::Arc; +use std::time::Duration; + +use async_nats::jetstream; +use async_nats::jetstream::consumer::pull::Config as PullConfig; +use async_nats::jetstream::stream::{Config as StreamConfig, Stream}; + +use super::nats::{NatsJetStreamSource, NatsPublisher}; +use super::{run_source, AsyncMessagePublisher, Bus, BusConsumer, RunOptions, TransportError}; +use crate::microsvc::{Message, MessageKind, Service}; + +const DEFAULT_FETCH_TIMEOUT: Duration = Duration::from_millis(500); + +fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { + TransportError::retryable(format!("{context}: {err}")) +} + +/// NATS JetStream [`Bus`] + [`BusConsumer`]. Cheap to clone. +#[derive(Clone)] +pub struct NatsBus { + jetstream: jetstream::Context, + cmd_publisher: Arc, + evt_publisher: Arc, + group: String, + namespace: String, + stream_name: String, + fetch_timeout: Duration, +} + +impl NatsBus { + /// Build a bus over an existing JetStream context. + /// + /// `group` is the logical consumer identity (same group ⇒ competing + /// consumers; different groups ⇒ fan-out). `namespace` scopes the stream and + /// subjects so multiple buses can share a server without collision. + pub fn new( + jetstream: jetstream::Context, + group: impl Into, + namespace: impl Into, + ) -> Self { + let namespace = namespace.into(); + let cmd_publisher = + NatsPublisher::new(jetstream.clone()).with_subject_prefix(format!("{namespace}.cmd")); + let evt_publisher = + NatsPublisher::new(jetstream.clone()).with_subject_prefix(format!("{namespace}.evt")); + Self { + jetstream, + cmd_publisher: Arc::new(cmd_publisher), + evt_publisher: Arc::new(evt_publisher), + group: group.into(), + stream_name: namespace.to_uppercase().replace(['.', '-'], "_"), + namespace, + fetch_timeout: DEFAULT_FETCH_TIMEOUT, + } + } + + /// Connect to a NATS server URL and build a bus. + pub async fn connect( + url: &str, + group: impl Into, + namespace: impl Into, + ) -> Result { + let client = async_nats::connect(url) + .await + .map_err(|err| retryable("nats connect", err))?; + Ok(Self::new(jetstream::new(client), group, namespace)) + } + + /// Override how long a `listen`/`subscribe` poll waits before idling. + pub fn with_fetch_timeout(mut self, timeout: Duration) -> Self { + self.fetch_timeout = timeout; + self + } + + /// Sanitize the group into a valid NATS consumer-name token. Consumer names + /// cannot contain `.`, `*`, `>`, or whitespace, so map them to `_`. + fn durable_base(&self) -> String { + self.group + .chars() + .map(|c| match c { + '.' | '*' | '>' | ' ' | '\t' | '\n' | '/' | '\\' => '_', + other => other, + }) + .collect() + } + + /// Create-or-open the backing stream (`{namespace}.>`). Called by + /// `listen`/`subscribe`; producers should ensure it exists (here, via IaC, or + /// by a consumer) before publishing, since JetStream rejects a publish to an + /// unbound subject. + pub async fn ensure_stream(&self) -> Result { + self.jetstream + .get_or_create_stream(StreamConfig { + name: self.stream_name.clone(), + subjects: vec![format!("{}.>", self.namespace)], + ..Default::default() + }) + .await + .map_err(|err| retryable("nats get_or_create_stream", err)) + } + + /// Build a durable pull source over the bus stream, filtered to `subjects`, + /// stripping `strip_prefix` so the dispatched message name is the bare name. + async fn source( + &self, + durable: &str, + subjects: Vec, + strip_prefix: String, + ) -> Result { + let stream = self.ensure_stream().await?; + let consumer = stream + .get_or_create_consumer( + durable, + PullConfig { + durable_name: Some(durable.to_string()), + filter_subjects: subjects, + ..Default::default() + }, + ) + .await + .map_err(|err| retryable("nats get_or_create_consumer", err))?; + Ok(NatsJetStreamSource::new(consumer) + .with_fetch_timeout(self.fetch_timeout) + .with_strip_prefix(strip_prefix)) + } +} + +impl Bus for NatsBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.send_message(Message::new(name, MessageKind::Command, payload)) + .await + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.publish_message(Message::new(name, MessageKind::Event, payload)) + .await + } + + async fn send_message(&self, message: Message) -> Result<(), TransportError> { + self.cmd_publisher.publish(message).await + } + + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.evt_publisher.publish(message).await + } +} + +impl BusConsumer for NatsBus { + async fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let subjects: Vec = service + .command_names() + .iter() + .map(|name| format!("{}.cmd.{name}", self.namespace)) + .collect(); + if subjects.is_empty() { + return Ok(()); + } + let source = self + .source( + &format!("{}_cmd", self.durable_base()), + subjects, + format!("{}.cmd.", self.namespace), + ) + .await?; + run_source(service, source, options).await + } + + async fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let subjects: Vec = service + .event_names() + .iter() + .map(|name| format!("{}.evt.{name}", self.namespace)) + .collect(); + if subjects.is_empty() { + return Ok(()); + } + let source = self + .source( + &format!("{}_evt", self.durable_base()), + subjects, + format!("{}.evt.", self.namespace), + ) + .await?; + run_source(service, source, options).await + } +} diff --git a/tests/nats_transport/main.rs b/tests/nats_transport/main.rs index 9060cc69b..37cc88876 100644 --- a/tests/nats_transport/main.rs +++ b/tests/nats_transport/main.rs @@ -10,12 +10,25 @@ use std::time::Duration; use serde_json::json; use sourced_rust::microsvc::transport::{ - run_source, AsyncMessagePublisher, NatsJetStreamSource, NatsPublisher, RunOptions, + run_source, AsyncMessagePublisher, Bus, BusConsumer, NatsBus, NatsJetStreamSource, + NatsPublisher, RunOptions, }; use sourced_rust::microsvc::{Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); +/// A token unique to this process run, so stream/consumer names don't collide +/// with state left by a previous run against the same server (the `SEQ` counter +/// resets each process). +fn run_token() -> u128 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + ^ u128::from(std::process::id()) +} + fn nats_url() -> Option { match std::env::var("NATS_URL") { Ok(url) => Some(url), @@ -26,9 +39,14 @@ fn nats_url() -> Option { } } -/// Unique subject/stream/durable per test so JetStream state does not collide. +/// Unique subject/stream/durable per test so JetStream state does not collide, +/// including across separate runs against a persistent server. fn unique(prefix: &str) -> String { - format!("{prefix}_{}", SEQ.fetch_add(1, Ordering::SeqCst)) + format!( + "{prefix}_{:x}_{}", + run_token(), + SEQ.fetch_add(1, Ordering::SeqCst) + ) } #[tokio::test] @@ -127,3 +145,115 @@ async fn message_id_and_metadata_survive_the_round_trip() { assert_eq!(got.1.as_deref(), Some("corr-9")); assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); } + +/// Build a service whose single handler records the message id into `rec`. +/// `kind` selects command vs event registration. +fn recording_service( + name: &str, + kind: MessageKind, + rec: Arc>>, +) -> Arc> { + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + let builder = Service::new(()); + let registered = match kind { + MessageKind::Command => builder.command(leaked), + MessageKind::Event => builder.event(leaked), + }; + Arc::new(registered.handle(move |ctx| { + rec.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })) +} + +/// `send` + `listen`: replicas sharing a `group` compete for the command — each +/// message is handled exactly once across the pool (point-to-point). +#[tokio::test] +async fn bus_send_listen_is_point_to_point_across_a_group() { + let Some(url) = nats_url() else { return }; + let namespace = unique("ns").to_lowercase(); + let group = "orders"; + + let producer = NatsBus::connect(&url, group, &namespace) + .await + .expect("connect producer") + .with_fetch_timeout(Duration::from_millis(600)); + producer.ensure_stream().await.expect("ensure stream"); + + let total = 6; + for i in 0..total { + producer + .send_message( + Message::new("work", MessageKind::Command, b"{}".to_vec()).with_id(format!("c{i}")), + ) + .await + .expect("send command"); + } + + // Two replicas of the same service (same group) drain concurrently. + let rec = Arc::new(Mutex::new(Vec::new())); + let bus_a = NatsBus::connect(&url, group, &namespace) + .await + .unwrap() + .with_fetch_timeout(Duration::from_millis(600)); + let bus_b = bus_a.clone(); + let svc_a = recording_service("work", MessageKind::Command, rec.clone()); + let svc_b = recording_service("work", MessageKind::Command, rec.clone()); + + let (ra, rb) = tokio::join!( + bus_a.listen(svc_a, RunOptions::idempotent()), + bus_b.listen(svc_b, RunOptions::idempotent()), + ); + ra.expect("replica a drains"); + rb.expect("replica b drains"); + + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + let expected: Vec = (0..total).map(|i| format!("c{i}")).collect(); + assert_eq!( + ids, expected, + "every command handled exactly once across the group" + ); +} + +/// `publish` + `subscribe`: distinct `group`s each get their own durable on the +/// shared stream, so every group sees every event (fan-out). +#[tokio::test] +async fn bus_publish_subscribe_fans_out_across_groups() { + let Some(url) = nats_url() else { return }; + let namespace = unique("ns").to_lowercase(); + + let producer = NatsBus::connect(&url, "publisher", &namespace) + .await + .expect("connect producer"); + producer.ensure_stream().await.expect("ensure stream"); + + let total = 4; + for i in 0..total { + producer + .publish_message( + Message::new("evt", MessageKind::Event, b"{}".to_vec()).with_id(format!("e{i}")), + ) + .await + .expect("publish event"); + } + + let expected: Vec = (0..total).map(|i| format!("e{i}")).collect(); + for group in ["projections", "audit"] { + let bus = NatsBus::connect(&url, group, &namespace) + .await + .unwrap() + .with_fetch_timeout(Duration::from_millis(600)); + let rec = Arc::new(Mutex::new(Vec::new())); + bus.subscribe( + recording_service("evt", MessageKind::Event, rec.clone()), + RunOptions::idempotent(), + ) + .await + .expect("subscriber drains"); + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + assert_eq!(ids, expected, "group {group} sees every event"); + } +} From 5e804673de1af1e84ebcf6449b3ed7d34cc7dd86 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:05:47 -0500 Subject: [PATCH 13/39] feat(transport): add PostgresBus (work queue + log/offset fan-out) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgresBus implements Bus + BusConsumer as a complete single-DB bus: - send/listen (point-to-point): a bus_queue work table claimed FOR UPDATE SKIP LOCKED under a lease, so replicas sharing a group compete — each command handled once (ack=delete, nack=release, dead_letter/park=delete). - publish/subscribe (fan-out): Postgres as a log — append-only bus_log (monotonic seq, retained) + per-consumer bus_offset (consumer → last_seq). publish appends; each group reads seq > last_seq for its event names in order and advances its own offset, so every group sees every event. ack advances the offset (the effectively-once point), nack leaves it for redelivery, dead_letter/park skips it. ensure_tables() provisions the three tables (mirrors NatsBus::ensure_stream). Implementation note: uses the spec's sanctioned claim-lease backend, not sqlxmq. Decision #8 listed sqlxmq as recommended with claim-lease as the no-dependency alternative; sqlxmq owns an always-on push JobRunner loop that doesn't compose with the facade's uniform drain-to-idle run_source model (a claim-lease source returns Ok(None) when empty and stops). The module header records this; sqlxmq stays a viable future backend. Two integration tests prove competing-across-a-group (each command once via concurrent replicas) and fan-out-across-groups (each group reads every event), against a live Postgres. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/mod.rs | 4 + src/microsvc/transport/postgres_bus.rs | 427 +++++++++++++++++++++++++ tests/postgres_transport/main.rs | 107 ++++++- 3 files changed, 536 insertions(+), 2 deletions(-) create mode 100644 src/microsvc/transport/postgres_bus.rs diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 83528cb16..54734e0b5 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -96,6 +96,8 @@ mod nats; mod nats_bus; mod outbox_dispatch; mod outbox_source; +#[cfg(feature = "postgres")] +mod postgres_bus; mod publisher; #[cfg(feature = "rabbitmq")] mod rabbitmq; @@ -124,6 +126,8 @@ pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METAD pub use outbox_source::{ OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, }; +#[cfg(feature = "postgres")] +pub use postgres_bus::{LogReceived, PostgresBus, QueueReceived}; pub use publisher::AsyncMessagePublisher; pub use run_options::{ConsumerDeliveryMode, InboxHook, NoInbox, RunOptions}; pub use runner::run_source; diff --git a/src/microsvc/transport/postgres_bus.rs b/src/microsvc/transport/postgres_bus.rs new file mode 100644 index 000000000..09b181231 --- /dev/null +++ b/src/microsvc/transport/postgres_bus.rs @@ -0,0 +1,427 @@ +//! Postgres [`Bus`] + [`BusConsumer`] — a complete single-DB bus. +//! +//! Postgres covers **both** bus modes (see [[specs/transport-bus-facade]]): +//! +//! - **`send` / `listen` (point-to-point, competing):** a durable work-queue +//! table (`bus_queue`) claimed with `FOR UPDATE SKIP LOCKED` under a lease, so +//! one of N competing `listen`ers handles each command and the row is deleted +//! on success (redelivered on nack, until a `dead_letter`/`park` drops it). +//! - **`publish` / `subscribe` (fan-out):** Postgres modelled as a log — an +//! append-only `bus_log` table (monotonic `seq`, retained) plus a per-consumer +//! offset table (`bus_offset`: `consumer → last_seq`). `publish` appends; each +//! `subscribe`r (keyed by its `group`) reads `seq > last_seq` for its event +//! names in order and advances its own offset, so every group sees every event. +//! Because the log, the offset, and projection writes share one Postgres, the +//! offset advances in the same database as the effects — the cleanest path to +//! transactional effectively-once of any transport (the offset is the inbox). +//! +//! ## Why claim-lease, not sqlxmq (implementation note) +//! +//! Decision #8 of the locked spec names `sqlxmq` as the recommended work-queue +//! backend and keeps the claim-lease queue as the **no-extra-dependency +//! alternative**. At implementation time the claim-lease queue was chosen because +//! sqlxmq owns an always-on, push-based `JobRunner` loop, which does not compose +//! with the facade's uniform *drain-to-idle* [`run_source`] model that every other +//! `*Bus` (in-memory, NATS, RabbitMQ, Kafka) and their tests share — a claim-lease +//! [`AsyncMessageSource`] returns `Ok(None)` when the queue is empty and stops +//! cleanly. sqlxmq remains a viable future backend for its mature NOTIFY/backoff; +//! revisit if those are needed. See [[tasks/build-transport-bus-facade]]. +//! +//! Requires the `postgres` feature. Integration-tested in `tests/postgres_transport`. +//! +//! [`run_source`]: super::run_source + +use std::sync::Arc; +use std::time::Duration; + +use sqlx::{PgPool, Row}; + +use super::source::{AsyncMessageSource, ReceivedMessage}; +use super::{run_source, Bus, BusConsumer, RunOptions, TransportError}; +use crate::microsvc::{Message, MessageKind, Service}; + +const DEFAULT_LEASE: Duration = Duration::from_secs(30); + +const SCHEMA: &str = "\ +CREATE TABLE IF NOT EXISTS bus_queue ( + seq BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL, + message_id TEXT, + kind TEXT NOT NULL, + payload BYTEA NOT NULL, + content_type TEXT NOT NULL DEFAULT 'application/json', + metadata TEXT NOT NULL DEFAULT '[]', + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + locked_until TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS bus_queue_claim_idx ON bus_queue (name, available_at, seq); +CREATE TABLE IF NOT EXISTS bus_log ( + seq BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL, + message_id TEXT, + kind TEXT NOT NULL, + payload BYTEA NOT NULL, + content_type TEXT NOT NULL DEFAULT 'application/json', + metadata TEXT NOT NULL DEFAULT '[]', + appended_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS bus_log_name_seq_idx ON bus_log (name, seq); +CREATE TABLE IF NOT EXISTS bus_offset ( + consumer TEXT PRIMARY KEY, + last_seq BIGINT NOT NULL DEFAULT 0 +)"; + +fn db_err(context: &str, err: sqlx::Error) -> TransportError { + // Database errors are transient from the transport's view; the runner retries. + TransportError::retryable(format!("postgres bus {context}: {err}")) +} + +fn kind_str(kind: MessageKind) -> &'static str { + match kind { + MessageKind::Command => "command", + MessageKind::Event => "event", + } +} + +fn kind_from_str(value: &str) -> MessageKind { + match value { + "command" => MessageKind::Command, + _ => MessageKind::Event, + } +} + +fn message_from_row(row: &sqlx::postgres::PgRow) -> Message { + let metadata_json: String = row.try_get("metadata").unwrap_or_else(|_| "[]".to_string()); + let metadata = + serde_json::from_str::>(&metadata_json).unwrap_or_default(); + Message { + id: row + .try_get::, _>("message_id") + .unwrap_or(None), + name: row.try_get("name").unwrap_or_default(), + kind: kind_from_str(&row.try_get::("kind").unwrap_or_default()), + payload: row.try_get("payload").unwrap_or_default(), + content_type: row + .try_get("content_type") + .unwrap_or_else(|_| "application/json".to_string()), + metadata, + } +} + +/// Postgres [`Bus`] + [`BusConsumer`]. Cheap to clone (the pool is an `Arc`). +#[derive(Clone)] +pub struct PostgresBus { + pool: PgPool, + group: String, + lease: Duration, +} + +impl PostgresBus { + /// Build a bus over an existing pool. `group` is the consumer identity: + /// replicas sharing a `group` compete on the queue (point-to-point) and share + /// one log offset; distinct `group`s each get their own log offset (fan-out). + pub fn new(pool: PgPool, group: impl Into) -> Self { + Self { + pool, + group: group.into(), + lease: DEFAULT_LEASE, + } + } + + /// Override the claim lease for `listen` (how long a claimed command stays + /// invisible to other workers before it is eligible for redelivery). + pub fn with_lease(mut self, lease: Duration) -> Self { + self.lease = lease; + self + } + + /// Create the bus tables (`bus_queue`, `bus_log`, `bus_offset`) if absent, in + /// the pool's current schema. Called by `listen`/`subscribe`; producers must + /// ensure the tables exist (here or via migration) before `send`/`publish`. + pub async fn ensure_tables(&self) -> Result<(), TransportError> { + for statement in SCHEMA.split(';') { + let statement = statement.trim(); + if statement.is_empty() { + continue; + } + sqlx::query(statement) + .execute(&self.pool) + .await + .map_err(|err| db_err("ensure_tables", err))?; + } + Ok(()) + } + + async fn enqueue(&self, message: Message) -> Result<(), TransportError> { + let metadata = serde_json::to_string(&message.metadata).unwrap_or_else(|_| "[]".into()); + sqlx::query( + "INSERT INTO bus_queue (name, message_id, kind, payload, content_type, metadata) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(&message.name) + .bind(&message.id) + .bind(kind_str(message.kind)) + .bind(&message.payload) + .bind(&message.content_type) + .bind(metadata) + .execute(&self.pool) + .await + .map_err(|err| db_err("enqueue", err))?; + Ok(()) + } + + async fn append(&self, message: Message) -> Result<(), TransportError> { + let metadata = serde_json::to_string(&message.metadata).unwrap_or_else(|_| "[]".into()); + sqlx::query( + "INSERT INTO bus_log (name, message_id, kind, payload, content_type, metadata) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(&message.name) + .bind(&message.id) + .bind(kind_str(message.kind)) + .bind(&message.payload) + .bind(&message.content_type) + .bind(metadata) + .execute(&self.pool) + .await + .map_err(|err| db_err("append", err))?; + Ok(()) + } +} + +impl Bus for PostgresBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.enqueue(Message::new(name, MessageKind::Command, payload)) + .await + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.append(Message::new(name, MessageKind::Event, payload)) + .await + } + + async fn send_message(&self, message: Message) -> Result<(), TransportError> { + self.enqueue(message).await + } + + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.append(message).await + } +} + +impl BusConsumer for PostgresBus { + async fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + self.ensure_tables().await?; + let names: Vec = service + .command_names() + .iter() + .map(|n| n.to_string()) + .collect(); + if names.is_empty() { + return Ok(()); + } + let source = QueueSource { + pool: self.pool.clone(), + names, + lease_secs: self.lease.as_secs_f64(), + }; + run_source(service, source, options).await + } + + async fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + self.ensure_tables().await?; + let names: Vec = service + .event_names() + .iter() + .map(|n| n.to_string()) + .collect(); + if names.is_empty() { + return Ok(()); + } + let source = LogSource { + pool: self.pool.clone(), + names, + consumer: self.group.clone(), + }; + run_source(service, source, options).await + } +} + +/// Competing-consumer source over `bus_queue` (`FOR UPDATE SKIP LOCKED` claim). +struct QueueSource { + pool: PgPool, + names: Vec, + lease_secs: f64, +} + +impl AsyncMessageSource for QueueSource { + type Received = QueueReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let row = sqlx::query( + "UPDATE bus_queue SET locked_until = now() + ($1 * interval '1 second'), \ + attempts = attempts + 1 \ + WHERE seq = ( \ + SELECT seq FROM bus_queue \ + WHERE name = ANY($2) AND available_at <= now() \ + AND (locked_until IS NULL OR locked_until < now()) \ + ORDER BY seq FOR UPDATE SKIP LOCKED LIMIT 1 \ + ) \ + RETURNING seq, name, message_id, kind, payload, content_type, metadata", + ) + .bind(self.lease_secs) + .bind(&self.names) + .fetch_optional(&self.pool) + .await + .map_err(|err| db_err("claim", err))?; + + Ok(row.map(|row| { + let seq: i64 = row.try_get("seq").unwrap_or_default(); + QueueReceived { + pool: self.pool.clone(), + seq, + message: message_from_row(&row), + } + })) + } +} + +/// A claimed `bus_queue` row: `ack` deletes it (done); `nack` makes it available +/// again (redelivery); `dead_letter`/`park` delete it (stop redelivery). +pub struct QueueReceived { + pool: PgPool, + seq: i64, + message: Message, +} + +impl QueueReceived { + async fn delete(&self) -> Result<(), TransportError> { + sqlx::query("DELETE FROM bus_queue WHERE seq = $1") + .bind(self.seq) + .execute(&self.pool) + .await + .map_err(|err| db_err("delete", err))?; + Ok(()) + } +} + +impl ReceivedMessage for QueueReceived { + fn message(&self) -> &Message { + &self.message + } + + async fn ack(self) -> Result<(), TransportError> { + self.delete().await + } + + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + sqlx::query("UPDATE bus_queue SET locked_until = NULL WHERE seq = $1") + .bind(self.seq) + .execute(&self.pool) + .await + .map_err(|err| db_err("nack", err))?; + Ok(()) + } + + async fn dead_letter(self, _reason: &str) -> Result<(), TransportError> { + self.delete().await + } + + async fn park(self, _reason: &str) -> Result<(), TransportError> { + self.delete().await + } +} + +/// Fan-out source over `bus_log`: reads the next entry past this consumer's +/// offset for its subscribed names, in `seq` order. +struct LogSource { + pool: PgPool, + names: Vec, + consumer: String, +} + +impl AsyncMessageSource for LogSource { + type Received = LogReceived; + + async fn recv(&mut self) -> Result, TransportError> { + let row = sqlx::query( + "SELECT seq, name, message_id, kind, payload, content_type, metadata FROM bus_log \ + WHERE name = ANY($1) \ + AND seq > COALESCE((SELECT last_seq FROM bus_offset WHERE consumer = $2), 0) \ + ORDER BY seq LIMIT 1", + ) + .bind(&self.names) + .bind(&self.consumer) + .fetch_optional(&self.pool) + .await + .map_err(|err| db_err("log read", err))?; + + Ok(row.map(|row| { + let seq: i64 = row.try_get("seq").unwrap_or_default(); + LogReceived { + pool: self.pool.clone(), + consumer: self.consumer.clone(), + seq, + message: message_from_row(&row), + } + })) + } +} + +/// A `bus_log` entry: `ack` advances this consumer's offset to its `seq` (the +/// effectively-once point); `nack` leaves the offset (redelivery); +/// `dead_letter`/`park` advance past it (skip, don't get stuck). +pub struct LogReceived { + pool: PgPool, + consumer: String, + seq: i64, + message: Message, +} + +impl LogReceived { + async fn advance_offset(&self) -> Result<(), TransportError> { + sqlx::query( + "INSERT INTO bus_offset (consumer, last_seq) VALUES ($1, $2) \ + ON CONFLICT (consumer) DO UPDATE SET last_seq = EXCLUDED.last_seq \ + WHERE bus_offset.last_seq < EXCLUDED.last_seq", + ) + .bind(&self.consumer) + .bind(self.seq) + .execute(&self.pool) + .await + .map_err(|err| db_err("advance offset", err))?; + Ok(()) + } +} + +impl ReceivedMessage for LogReceived { + fn message(&self) -> &Message { + &self.message + } + + async fn ack(self) -> Result<(), TransportError> { + self.advance_offset().await + } + + async fn nack(self, _reason: &str) -> Result<(), TransportError> { + // Leave the offset unmoved so the entry is re-read on the next poll. + Ok(()) + } + + async fn dead_letter(self, _reason: &str) -> Result<(), TransportError> { + self.advance_offset().await + } + + async fn park(self, _reason: &str) -> Result<(), TransportError> { + self.advance_offset().await + } +} diff --git a/tests/postgres_transport/main.rs b/tests/postgres_transport/main.rs index 3b967bf1f..7f084e1ba 100644 --- a/tests/postgres_transport/main.rs +++ b/tests/postgres_transport/main.rs @@ -13,9 +13,10 @@ use std::sync::{Arc, Mutex}; use serde_json::json; use sourced_rust::microsvc::transport::{ - run_source, AsyncMessageSource, OutboxSource, ReceivedMessage, RunOptions, + run_source, AsyncMessageSource, Bus, BusConsumer, OutboxSource, PostgresBus, ReceivedMessage, + RunOptions, }; -use sourced_rust::microsvc::Service; +use sourced_rust::microsvc::{Message, MessageKind, Service}; use sourced_rust::{ AsyncCommitBatch, AsyncOutboxStore, AsyncTransactionalCommit, OutboxMessage, OutboxMessageStatus, PostgresOutboxStore, PostgresRepository, @@ -183,3 +184,105 @@ async fn dead_letter_marks_row_failed() { Some(OutboxMessageStatus::Failed) ); } + +// ---- PostgresBus: send/listen (work queue) + publish/subscribe (log+offsets) ---- + +/// Service whose single handler records the message id; `kind` picks command vs +/// event registration. +fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + let builder = Service::new(()); + let registered = match kind { + MessageKind::Command => builder.command(leaked), + MessageKind::Event => builder.event(leaked), + }; + Arc::new(registered.handle(move |ctx| { + rec.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })) +} + +/// `send` + `listen`: the work queue is claimed `FOR UPDATE SKIP LOCKED`, so two +/// replicas sharing a `group` compete — each command handled exactly once. +#[tokio::test] +async fn bus_send_listen_is_point_to_point_across_a_group() { + let Some(schema) = postgres::PostgresTestSchema::create_from_env("bus_pp", SKIP).await else { + return; + }; + let repo = schema.repository().await; + let bus = PostgresBus::new(repo.pool().clone(), "orders"); + bus.ensure_tables().await.expect("ensure tables"); + + let total = 6; + for i in 0..total { + bus.send_message( + Message::new("work", MessageKind::Command, b"{}".to_vec()).with_id(format!("c{i}")), + ) + .await + .expect("send command"); + } + + let rec = Arc::new(Mutex::new(Vec::new())); + let bus_a = bus.clone(); + let bus_b = bus.clone(); + let (ra, rb) = tokio::join!( + bus_a.listen( + recording_for("work", MessageKind::Command, rec.clone()), + RunOptions::idempotent() + ), + bus_b.listen( + recording_for("work", MessageKind::Command, rec.clone()), + RunOptions::idempotent() + ), + ); + ra.expect("replica a drains"); + rb.expect("replica b drains"); + + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + let expected: Vec = (0..total).map(|i| format!("c{i}")).collect(); + assert_eq!( + ids, expected, + "every command handled exactly once across the group" + ); +} + +/// `publish` + `subscribe`: each `group` has its own log offset, so every group +/// reads the full log — fan-out. nacked entries do not advance the offset. +#[tokio::test] +async fn bus_publish_subscribe_fans_out_across_groups() { + let Some(schema) = postgres::PostgresTestSchema::create_from_env("bus_fan", SKIP).await else { + return; + }; + let repo = schema.repository().await; + let pool = repo.pool().clone(); + let producer = PostgresBus::new(pool.clone(), "producer"); + producer.ensure_tables().await.expect("ensure tables"); + + let total = 4; + for i in 0..total { + producer + .publish_message( + Message::new("evt", MessageKind::Event, b"{}".to_vec()).with_id(format!("e{i}")), + ) + .await + .expect("publish event"); + } + + let expected: Vec = (0..total).map(|i| format!("e{i}")).collect(); + for group in ["projections", "audit"] { + let bus = PostgresBus::new(pool.clone(), group); + let rec = Arc::new(Mutex::new(Vec::new())); + bus.subscribe( + recording_for("evt", MessageKind::Event, rec.clone()), + RunOptions::idempotent(), + ) + .await + .expect("subscriber drains"); + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + assert_eq!(ids, expected, "group {group} sees every event"); + } +} From 0121e453bfb1019956f72d631f141fd692075bb6 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:13:53 -0500 Subject: [PATCH 14/39] feat(transport): add RabbitBus (send/listen + publish/subscribe) RabbitBus implements Bus + BusConsumer over two AMQP exchange shapes: - send/listen (point-to-point): the default exchange routes to a durable queue {ns}.cmd.{name}; send declares the queue and publishes with a publisher confirm. Replicas sharing a queue compete (AMQP round-robin). - publish/subscribe (fan-out): a durable topic exchange {ns}.events; publish routes by event name; each subscriber declares its own queue {ns}.evt.{group} bound to the exchange for its event names, so every group receives every event. The message name is resolved from the delivery routing key (stripping the {ns}.cmd. prefix for commands). Exposes pub(super) connect_channel / message_properties / RabbitReceived::from_delivery_with_name from the adapter, and a public RabbitBus::ensure_subscription so a producer can bind all subscriber queues before publishing (a topic exchange drops events with no matching binding). Two integration tests prove competing-across-a-group and fan-out-across-groups against a live broker. Makes tests/rabbitmq_transport unique() process-unique so durable queue/exchange names don't collide across runs. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/mod.rs | 4 + src/microsvc/transport/rabbit_bus.rs | 278 +++++++++++++++++++++++++++ src/microsvc/transport/rabbitmq.rs | 16 +- tests/rabbitmq_transport/main.rs | 141 +++++++++++++- 4 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 src/microsvc/transport/rabbit_bus.rs diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 54734e0b5..ad2ddcf6e 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -100,6 +100,8 @@ mod outbox_source; mod postgres_bus; mod publisher; #[cfg(feature = "rabbitmq")] +mod rabbit_bus; +#[cfg(feature = "rabbitmq")] mod rabbitmq; mod run_options; mod runner; @@ -115,6 +117,8 @@ pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; #[cfg(feature = "nats")] pub use nats_bus::NatsBus; #[cfg(feature = "rabbitmq")] +pub use rabbit_bus::RabbitBus; +#[cfg(feature = "rabbitmq")] pub use rabbitmq::{RabbitPublisher, RabbitReceived, RabbitSource}; pub use bus::{Bus, BusConsumer}; diff --git a/src/microsvc/transport/rabbit_bus.rs b/src/microsvc/transport/rabbit_bus.rs new file mode 100644 index 000000000..f1407178c --- /dev/null +++ b/src/microsvc/transport/rabbit_bus.rs @@ -0,0 +1,278 @@ +//! RabbitMQ (AMQP 0-9-1) [`Bus`] + [`BusConsumer`]. +//! +//! RabbitMQ shines through its exchange topologies, so the two bus surfaces map +//! onto two different exchange shapes: +//! +//! - **`send` / `listen` (point-to-point, competing):** the **default exchange** +//! routes a message to the queue named by its routing key. `send(name)` +//! declares a durable queue `{ns}.cmd.{name}` and publishes to it; `listen` +//! consumes those queues. Replicas sharing a queue compete (AMQP round-robins +//! across consumers) — point-to-point. +//! - **`publish` / `subscribe` (fan-out):** a durable **topic exchange** +//! `{ns}.events`. `publish(name)` publishes to it with routing key `name`; each +//! `subscribe`r (keyed by `group`) declares its **own** queue `{ns}.evt.{group}` +//! bound to the exchange for its event names. Distinct groups get distinct +//! queues, so every group receives every event — fan-out (replicas within a +//! group still compete on that group's queue). +//! +//! The `group` is the logical consumer identity. `{ns}` (namespace) scopes queue +//! and exchange names so multiple apps can share a broker. +//! +//! Requires the `rabbitmq` feature. Integration-tested in `tests/rabbitmq_transport`. + +use std::sync::Arc; + +use lapin::options::{ + BasicGetOptions, BasicPublishOptions, ConfirmSelectOptions, ExchangeDeclareOptions, + QueueBindOptions, QueueDeclareOptions, +}; +use lapin::types::FieldTable; +use lapin::{Channel, ExchangeKind}; + +use super::rabbitmq::{connect_channel, message_properties, RabbitReceived}; +use super::source::AsyncMessageSource; +use super::{run_source, Bus, BusConsumer, RunOptions, TransportError}; +use crate::microsvc::{Message, MessageKind, Service}; + +fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { + TransportError::retryable(format!("{context}: {err}")) +} + +/// RabbitMQ [`Bus`] + [`BusConsumer`]. +pub struct RabbitBus { + uri: String, + channel: Channel, + group: String, + namespace: String, + events_exchange: String, +} + +impl RabbitBus { + /// Connect to an AMQP URI and build a bus. `group` is the consumer identity + /// (same group ⇒ competing; different groups ⇒ fan-out); `namespace` scopes + /// queue/exchange names. + pub async fn connect( + uri: &str, + group: impl Into, + namespace: impl Into, + ) -> Result { + let channel = connect_channel(uri).await?; + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|err| retryable("amqp confirm_select", err))?; + let namespace = namespace.into(); + Ok(Self { + uri: uri.to_string(), + channel, + group: group.into(), + events_exchange: format!("{namespace}.events"), + namespace, + }) + } + + fn command_queue(&self, name: &str) -> String { + format!("{}.cmd.{name}", self.namespace) + } + + fn command_prefix(&self) -> String { + format!("{}.cmd.", self.namespace) + } + + fn group_queue(&self) -> String { + format!("{}.evt.{}", self.namespace, self.group) + } + + async fn declare_queue(&self, channel: &Channel, queue: &str) -> Result<(), TransportError> { + channel + .queue_declare( + queue, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|err| retryable("amqp queue_declare", err))?; + Ok(()) + } + + async fn declare_events_exchange(&self, channel: &Channel) -> Result<(), TransportError> { + channel + .exchange_declare( + &self.events_exchange, + ExchangeKind::Topic, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|err| retryable("amqp exchange_declare", err))?; + Ok(()) + } + + async fn publish_confirmed( + &self, + exchange: &str, + routing_key: &str, + message: &Message, + ) -> Result<(), TransportError> { + let confirm = self + .channel + .basic_publish( + exchange, + routing_key, + BasicPublishOptions::default(), + &message.payload, + message_properties(message), + ) + .await + .map_err(|err| retryable("amqp publish", err))?; + if confirm + .await + .map_err(|err| retryable("amqp publisher confirm", err))? + .is_nack() + { + return Err(TransportError::retryable("amqp publisher confirm: nack")); + } + Ok(()) + } + + /// Declare the topic exchange, this group's queue, and bind it to the + /// service's event names — the durable setup `subscribe` needs. Exposed so a + /// producer can ensure all subscriber bindings exist *before* publishing + /// (RabbitMQ drops events with no matching binding). + pub async fn ensure_subscription( + &self, + service: &Service, + ) -> Result<(), TransportError> { + self.declare_events_exchange(&self.channel).await?; + let queue = self.group_queue(); + self.declare_queue(&self.channel, &queue).await?; + for name in service.event_names() { + self.channel + .queue_bind( + &queue, + &self.events_exchange, + name, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|err| retryable("amqp queue_bind", err))?; + } + Ok(()) + } +} + +impl Bus for RabbitBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.send_message(Message::new(name, MessageKind::Command, payload)) + .await + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.publish_message(Message::new(name, MessageKind::Event, payload)) + .await + } + + async fn send_message(&self, mut message: Message) -> Result<(), TransportError> { + // Default exchange routes by routing key == queue name; declare the queue + // so the command is retained until a listener consumes it. + let queue = self.command_queue(message.name()); + self.declare_queue(&self.channel, &queue).await?; + message.name = queue.clone(); + self.publish_confirmed("", &queue, &message).await + } + + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.declare_events_exchange(&self.channel).await?; + let routing_key = message.name().to_string(); + self.publish_confirmed(&self.events_exchange, &routing_key, &message) + .await + } +} + +impl BusConsumer for RabbitBus { + async fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let channel = connect_channel(&self.uri).await?; + let mut queues = Vec::new(); + for name in service.command_names() { + let queue = self.command_queue(name); + self.declare_queue(&channel, &queue).await?; + queues.push(queue); + } + if queues.is_empty() { + return Ok(()); + } + let source = RabbitBusSource { + channel, + queues, + strip_prefix: Some(self.command_prefix()), + }; + run_source(service, source, options).await + } + + async fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + self.ensure_subscription(&service).await?; + if service.event_names().is_empty() { + return Ok(()); + } + let channel = connect_channel(&self.uri).await?; + let source = RabbitBusSource { + channel, + queues: vec![self.group_queue()], + // Events are published with routing key == the bare event name. + strip_prefix: None, + }; + run_source(service, source, options).await + } +} + +/// Polls one or more queues with `basic_get`, resolving the message name from the +/// delivery's routing key (stripping `strip_prefix` for command queues). +struct RabbitBusSource { + channel: Channel, + queues: Vec, + strip_prefix: Option, +} + +impl AsyncMessageSource for RabbitBusSource { + type Received = RabbitReceived; + + async fn recv(&mut self) -> Result, TransportError> { + for queue in &self.queues { + let got = self + .channel + .basic_get(queue, BasicGetOptions::default()) + .await + .map_err(|err| retryable("amqp basic_get", err))?; + if let Some(get) = got { + let routing_key = get.delivery.routing_key.to_string(); + let name = match &self.strip_prefix { + Some(prefix) => routing_key + .strip_prefix(prefix.as_str()) + .unwrap_or(&routing_key) + .to_string(), + None => routing_key, + }; + return Ok(Some(RabbitReceived::from_delivery_with_name( + get.delivery, + name, + ))); + } + } + Ok(None) + } +} diff --git a/src/microsvc/transport/rabbitmq.rs b/src/microsvc/transport/rabbitmq.rs index 6c4dc622a..d2df99029 100644 --- a/src/microsvc/transport/rabbitmq.rs +++ b/src/microsvc/transport/rabbitmq.rs @@ -28,7 +28,7 @@ fn retryable(context: &str, err: impl std::fmt::Display) -> TransportError { TransportError::retryable(format!("{context}: {err}")) } -async fn connect_channel(uri: &str) -> Result { +pub(super) async fn connect_channel(uri: &str) -> Result { let connection = Connection::connect(uri, ConnectionProperties::default()) .await .map_err(|err| retryable("amqp connect", err))?; @@ -60,7 +60,7 @@ impl RabbitPublisher { } } -fn message_properties(message: &Message) -> BasicProperties { +pub(super) fn message_properties(message: &Message) -> BasicProperties { let mut headers = FieldTable::default(); headers.insert( ShortString::from(MESSAGE_KIND_HEADER), @@ -158,6 +158,15 @@ pub struct RabbitReceived { impl RabbitReceived { fn from_delivery(delivery: Delivery, queue: String) -> Self { + // The standalone source consumes a queue named for the message, so the + // queue name is the routed message name. + Self::from_delivery_with_name(delivery, queue) + } + + /// Build from a delivery with an explicitly resolved message name. Used by + /// [`RabbitBus`](super::RabbitBus), which derives the name from the routing + /// key (stripping its `{ns}.cmd.` prefix for commands) rather than the queue. + pub(super) fn from_delivery_with_name(delivery: Delivery, name: String) -> Self { let payload = delivery.data.clone(); let id = delivery .properties @@ -177,8 +186,7 @@ impl RabbitReceived { } } } - // The queue name is the routed message name. - let mut message = Message::new(queue, kind, payload); + let mut message = Message::new(name, kind, payload); message.id = id; message.metadata = metadata; Self { delivery, message } diff --git a/tests/rabbitmq_transport/main.rs b/tests/rabbitmq_transport/main.rs index 2762a165a..1947ffa36 100644 --- a/tests/rabbitmq_transport/main.rs +++ b/tests/rabbitmq_transport/main.rs @@ -9,12 +9,24 @@ use std::sync::{Arc, Mutex}; use serde_json::json; use sourced_rust::microsvc::transport::{ - run_source, AsyncMessagePublisher, RabbitPublisher, RabbitSource, RunOptions, + run_source, AsyncMessagePublisher, Bus, BusConsumer, RabbitBus, RabbitPublisher, RabbitSource, + RunOptions, }; use sourced_rust::microsvc::{Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); +/// A token unique to this process run, so durable queue/exchange names don't +/// collide with state left by a previous run against the same broker. +fn run_token() -> u128 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + ^ u128::from(std::process::id()) +} + fn amqp_url() -> Option { match std::env::var("AMQP_URL") { Ok(url) => Some(url), @@ -26,7 +38,28 @@ fn amqp_url() -> Option { } fn unique(prefix: &str) -> String { - format!("{prefix}_{}", SEQ.fetch_add(1, Ordering::SeqCst)) + format!( + "{prefix}_{:x}_{}", + run_token(), + SEQ.fetch_add(1, Ordering::SeqCst) + ) +} + +/// Service whose single handler records the message id; `kind` picks command vs +/// event registration. +fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + let builder = Service::new(()); + let registered = match kind { + MessageKind::Command => builder.command(leaked), + MessageKind::Event => builder.event(leaked), + }; + Arc::new(registered.handle(move |ctx| { + rec.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })) } #[tokio::test] @@ -116,3 +149,107 @@ async fn message_id_and_metadata_survive_the_round_trip() { assert_eq!(got.1.as_deref(), Some("corr-9")); assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); } + +// ---- RabbitBus: send/listen (default exchange) + publish/subscribe (topic exchange) ---- + +/// `send` + `listen`: a durable command queue is shared by replicas of a group, +/// so AMQP round-robins — each command handled exactly once (point-to-point). +#[tokio::test] +async fn bus_send_listen_is_point_to_point_across_a_group() { + let Some(url) = amqp_url() else { return }; + let ns = unique("ns").to_lowercase(); + + let producer = RabbitBus::connect(&url, "orders", &ns) + .await + .expect("connect producer"); + let total = 6; + for i in 0..total { + producer + .send_message( + Message::new("work", MessageKind::Command, b"{}".to_vec()).with_id(format!("c{i}")), + ) + .await + .expect("send command"); + } + + // Two replicas of the same group (separate connections) drain concurrently. + let rec = Arc::new(Mutex::new(Vec::new())); + let bus_a = RabbitBus::connect(&url, "orders", &ns).await.unwrap(); + let bus_b = RabbitBus::connect(&url, "orders", &ns).await.unwrap(); + let (ra, rb) = tokio::join!( + bus_a.listen( + recording_for("work", MessageKind::Command, rec.clone()), + RunOptions::idempotent() + ), + bus_b.listen( + recording_for("work", MessageKind::Command, rec.clone()), + RunOptions::idempotent() + ), + ); + ra.expect("replica a drains"); + rb.expect("replica b drains"); + + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + let expected: Vec = (0..total).map(|i| format!("c{i}")).collect(); + assert_eq!( + ids, expected, + "every command handled exactly once across the group" + ); +} + +/// `publish` + `subscribe`: each group binds its own queue to the topic exchange, +/// so every group receives every event (fan-out). +#[tokio::test] +async fn bus_publish_subscribe_fans_out_across_groups() { + let Some(url) = amqp_url() else { return }; + let ns = unique("ns").to_lowercase(); + + let producer = RabbitBus::connect(&url, "producer", &ns) + .await + .expect("connect producer"); + + // Bind both subscribers' queues BEFORE publishing — the topic exchange drops + // events with no matching binding. + let proj_rec = Arc::new(Mutex::new(Vec::new())); + let audit_rec = Arc::new(Mutex::new(Vec::new())); + let svc_proj = recording_for("evt", MessageKind::Event, proj_rec.clone()); + let svc_audit = recording_for("evt", MessageKind::Event, audit_rec.clone()); + let bus_proj = RabbitBus::connect(&url, "projections", &ns).await.unwrap(); + let bus_audit = RabbitBus::connect(&url, "audit", &ns).await.unwrap(); + bus_proj + .ensure_subscription(&svc_proj) + .await + .expect("bind proj"); + bus_audit + .ensure_subscription(&svc_audit) + .await + .expect("bind audit"); + + let total = 4; + for i in 0..total { + producer + .publish_message( + Message::new("evt", MessageKind::Event, b"{}".to_vec()).with_id(format!("e{i}")), + ) + .await + .expect("publish event"); + } + + bus_proj + .subscribe(svc_proj, RunOptions::idempotent()) + .await + .expect("proj drains"); + bus_audit + .subscribe(svc_audit, RunOptions::idempotent()) + .await + .expect("audit drains"); + + let expected: Vec = (0..total).map(|i| format!("e{i}")).collect(); + let mut proj_ids = proj_rec.lock().unwrap().clone(); + let mut audit_ids = audit_rec.lock().unwrap().clone(); + proj_ids.sort(); + audit_ids.sort(); + assert_eq!(proj_ids, expected, "projections sees every event"); + assert_eq!(audit_ids, expected, "audit sees every event"); +} From 14de16a628428087a1039fea96b590b443fbc94c Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:20:36 -0500 Subject: [PATCH 15/39] feat(transport): add KafkaBus (send/listen + publish/subscribe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KafkaBus implements Bus + BusConsumer; point-to-point vs fan-out is a consumer-group choice: - send/listen (point-to-point): commands → topics {ns}.cmd.{name}; listen joins a shared group {ns}.{group}.cmd, so Kafka distributes partitions across members — each record handled by one replica. - publish/subscribe (fan-out): events → topics {ns}.evt.{name}; subscribe joins a per-service group {ns}.{group}.evt. Kafka delivers every record to every group, so each distinct group sees every event. Adds KafkaSource::with_strip_prefix (default off) so the dispatched message name is the topic minus its {ns}.cmd./{ns}.evt. prefix. Two integration tests against a live broker: point-to-point is proven deterministically (a second consumer in the same group reads nothing — the group's offset is committed past the end), which avoids the rebalance-redelivery flakiness a concurrent two-replica race would have; fan-out is proven across two distinct groups each reading from earliest. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/kafka.rs | 18 +++- src/microsvc/transport/kafka_bus.rs | 147 ++++++++++++++++++++++++++++ src/microsvc/transport/mod.rs | 4 + tests/kafka_transport/main.rs | 120 ++++++++++++++++++++++- 4 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 src/microsvc/transport/kafka_bus.rs diff --git a/src/microsvc/transport/kafka.rs b/src/microsvc/transport/kafka.rs index 39233dc05..1c5db8b47 100644 --- a/src/microsvc/transport/kafka.rs +++ b/src/microsvc/transport/kafka.rs @@ -98,6 +98,7 @@ impl AsyncMessagePublisher for KafkaPublisher { pub struct KafkaSource { consumer: Arc, fetch_timeout: Duration, + strip_prefix: Option, } impl KafkaSource { @@ -106,6 +107,7 @@ impl KafkaSource { Self { consumer, fetch_timeout: Duration::from_secs(5), + strip_prefix: None, } } @@ -115,6 +117,14 @@ impl KafkaSource { self } + /// Strip `prefix` from each record's topic when deriving the message name, so + /// a topic `app.cmd.account.debit` becomes the name `account.debit`. Used by + /// [`KafkaBus`](super::KafkaBus); default: no stripping (the topic is the name). + pub fn with_strip_prefix(mut self, prefix: impl Into) -> Self { + self.strip_prefix = Some(prefix.into()); + self + } + /// Connect a consumer (group `group_id`, auto-commit off, earliest reset) and /// subscribe to `topics`. pub async fn connect( @@ -156,6 +166,7 @@ impl AsyncMessageSource for KafkaSource { return Ok(Some(KafkaReceived::from_borrowed( &borrowed, self.consumer.clone(), + self.strip_prefix.as_deref(), ))); } Ok(Err(_transient)) => { @@ -181,9 +192,14 @@ impl KafkaReceived { fn from_borrowed( borrowed: &rdkafka::message::BorrowedMessage<'_>, consumer: Arc, + strip_prefix: Option<&str>, ) -> Self { let payload = borrowed.payload().map(|p| p.to_vec()).unwrap_or_default(); let topic = borrowed.topic().to_string(); + let name = match strip_prefix { + Some(prefix) => topic.strip_prefix(prefix).unwrap_or(&topic).to_string(), + None => topic.clone(), + }; let mut id = None; let mut kind = MessageKind::Event; let mut metadata = Vec::new(); @@ -200,7 +216,7 @@ impl KafkaReceived { } } } - let mut message = Message::new(topic.clone(), kind, payload); + let mut message = Message::new(name, kind, payload); message.id = id; message.metadata = metadata; Self { diff --git a/src/microsvc/transport/kafka_bus.rs b/src/microsvc/transport/kafka_bus.rs new file mode 100644 index 000000000..ccabb4a35 --- /dev/null +++ b/src/microsvc/transport/kafka_bus.rs @@ -0,0 +1,147 @@ +//! Kafka [`Bus`] + [`BusConsumer`]. +//! +//! Kafka shines as a partitioned, replayable log, and the point-to-point vs +//! fan-out distinction is entirely a **consumer-group** choice: +//! +//! - **`send` / `listen` (point-to-point, competing):** commands go to topics +//! `{ns}.cmd.{name}`. `listen` joins a **shared** consumer group +//! `{ns}.{group}.cmd`, so Kafka distributes the topic partitions across the +//! group's members — each record is handled by exactly one replica. +//! - **`publish` / `subscribe` (fan-out):** events go to topics `{ns}.evt.{name}`. +//! `subscribe` joins a group **per service** (`{ns}.{group}.evt`). Kafka +//! delivers every record to every group, so each distinct `group` sees every +//! event (replicas within a group still share its partitions → competing). +//! +//! The dispatched message name is the topic with its `{ns}.cmd.`/`{ns}.evt.` +//! prefix stripped. `{ns}` (namespace) scopes topics and groups so runs/apps +//! don't collide. +//! +//! Requires the `kafka` feature. Integration-tested in `tests/kafka_transport`. + +use std::sync::Arc; +use std::time::Duration; + +use super::kafka::{KafkaPublisher, KafkaSource}; +use super::{run_source, AsyncMessagePublisher, Bus, BusConsumer, RunOptions, TransportError}; +use crate::microsvc::{Message, MessageKind, Service}; + +const DEFAULT_FETCH_TIMEOUT: Duration = Duration::from_secs(8); + +/// Kafka [`Bus`] + [`BusConsumer`]. Cheap to clone. +#[derive(Clone)] +pub struct KafkaBus { + brokers: String, + publisher: Arc, + group: String, + namespace: String, + fetch_timeout: Duration, +} + +impl KafkaBus { + /// Connect a producer to `brokers` and build a bus. `group` is the consumer + /// identity (same group ⇒ competing; different groups ⇒ fan-out); `namespace` + /// scopes topics and group ids. + pub async fn connect( + brokers: &str, + group: impl Into, + namespace: impl Into, + ) -> Result { + let publisher = KafkaPublisher::connect(brokers).await?; + Ok(Self { + brokers: brokers.to_string(), + publisher: Arc::new(publisher), + group: group.into(), + namespace: namespace.into(), + fetch_timeout: DEFAULT_FETCH_TIMEOUT, + }) + } + + /// Override how long a `listen`/`subscribe` poll waits before idling. Kafka + /// group bootstrap/rebalance takes time, so this is generous by default. + pub fn with_fetch_timeout(mut self, timeout: Duration) -> Self { + self.fetch_timeout = timeout; + self + } + + fn command_prefix(&self) -> String { + format!("{}.cmd.", self.namespace) + } + + fn event_prefix(&self) -> String { + format!("{}.evt.", self.namespace) + } + + async fn run( + &self, + service: Arc>, + topics: Vec, + group_id: String, + strip_prefix: String, + options: RunOptions, + ) -> Result<(), TransportError> { + if topics.is_empty() { + return Ok(()); + } + let topic_refs: Vec<&str> = topics.iter().map(String::as_str).collect(); + let source = KafkaSource::connect(&self.brokers, &group_id, &topic_refs) + .await? + .with_fetch_timeout(self.fetch_timeout) + .with_strip_prefix(strip_prefix); + run_source(service, source, options).await + } +} + +impl Bus for KafkaBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.send_message(Message::new(name, MessageKind::Command, payload)) + .await + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.publish_message(Message::new(name, MessageKind::Event, payload)) + .await + } + + async fn send_message(&self, mut message: Message) -> Result<(), TransportError> { + // The publisher uses the message name as the topic; namespace it. + message.name = format!("{}{}", self.command_prefix(), message.name); + self.publisher.publish(message).await + } + + async fn publish_message(&self, mut message: Message) -> Result<(), TransportError> { + message.name = format!("{}{}", self.event_prefix(), message.name); + self.publisher.publish(message).await + } +} + +impl BusConsumer for KafkaBus { + async fn listen( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let prefix = self.command_prefix(); + let topics: Vec = service + .command_names() + .iter() + .map(|name| format!("{prefix}{name}")) + .collect(); + let group_id = format!("{}.{}.cmd", self.namespace, self.group); + self.run(service, topics, group_id, prefix, options).await + } + + async fn subscribe( + &self, + service: Arc>, + options: RunOptions, + ) -> Result<(), TransportError> { + let prefix = self.event_prefix(); + let topics: Vec = service + .event_names() + .iter() + .map(|name| format!("{prefix}{name}")) + .collect(); + let group_id = format!("{}.{}.evt", self.namespace, self.group); + self.run(service, topics, group_id, prefix, options).await + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index ad2ddcf6e..6f0815a04 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -88,6 +88,8 @@ mod failure_policy; mod in_memory_bus; #[cfg(feature = "kafka")] mod kafka; +#[cfg(feature = "kafka")] +mod kafka_bus; #[cfg(feature = "http")] mod knative; #[cfg(feature = "nats")] @@ -110,6 +112,8 @@ mod stable_id; #[cfg(feature = "kafka")] pub use kafka::{KafkaPublisher, KafkaReceived, KafkaSource}; +#[cfg(feature = "kafka")] +pub use kafka_bus::KafkaBus; #[cfg(feature = "http")] pub use knative::{cloud_events_router, knative_triggers}; #[cfg(feature = "nats")] diff --git a/tests/kafka_transport/main.rs b/tests/kafka_transport/main.rs index 3e6aecc9a..a801f2703 100644 --- a/tests/kafka_transport/main.rs +++ b/tests/kafka_transport/main.rs @@ -11,12 +11,30 @@ use std::time::Duration; use serde_json::json; use sourced_rust::microsvc::transport::{ - run_source, AsyncMessagePublisher, KafkaPublisher, KafkaSource, RunOptions, + run_source, AsyncMessagePublisher, Bus, BusConsumer, KafkaBus, KafkaPublisher, KafkaSource, + RunOptions, }; use sourced_rust::microsvc::{Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); +/// Service whose single handler records the message id; `kind` picks command vs +/// event registration. +fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + let builder = Service::new(()); + let registered = match kind { + MessageKind::Command => builder.command(leaked), + MessageKind::Event => builder.event(leaked), + }; + Arc::new(registered.handle(move |ctx| { + rec.lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + Ok(json!({})) + })) +} + fn brokers() -> Option { match std::env::var("KAFKA_BROKERS") { Ok(b) => Some(b), @@ -126,3 +144,103 @@ async fn message_id_and_metadata_survive_the_round_trip() { assert_eq!(got.1.as_deref(), Some("corr-9")); assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); } + +// ---- KafkaBus: shared group = listen (point-to-point); group-per-service = subscribe (fan-out) ---- + +/// `send` + `listen`: a shared consumer group consumes each command once for the +/// group as a whole. Proven deterministically: a first listener drains every +/// command; a second listener in the **same group** then reads nothing, because +/// the group's offset is already committed past the end — point-to-point. +#[tokio::test] +async fn bus_listen_shared_group_consumes_each_command_once() { + let Some(brokers) = brokers() else { return }; + let ns = unique("ns"); + + let producer = KafkaBus::connect(&brokers, "orders", &ns) + .await + .expect("connect producer"); + let total = 5; + for i in 0..total { + producer + .send_message( + Message::new("work", MessageKind::Command, b"{}".to_vec()).with_id(format!("c{i}")), + ) + .await + .expect("send command"); + } + + // First member of group "orders" drains every command. + let first = Arc::new(Mutex::new(Vec::new())); + KafkaBus::connect(&brokers, "orders", &ns) + .await + .unwrap() + .with_fetch_timeout(Duration::from_secs(10)) + .listen( + recording_for("work", MessageKind::Command, first.clone()), + RunOptions::idempotent(), + ) + .await + .expect("first listener drains"); + let mut ids = first.lock().unwrap().clone(); + ids.sort(); + let expected: Vec = (0..total).map(|i| format!("c{i}")).collect(); + assert_eq!(ids, expected, "the group consumes every command"); + + // A second member of the SAME group sees nothing — the group already consumed + // and committed past these records (point-to-point, not fan-out). + let second = Arc::new(Mutex::new(Vec::new())); + KafkaBus::connect(&brokers, "orders", &ns) + .await + .unwrap() + .with_fetch_timeout(Duration::from_secs(6)) + .listen( + recording_for("work", MessageKind::Command, second.clone()), + RunOptions::idempotent(), + ) + .await + .expect("second listener drains"); + assert!( + second.lock().unwrap().is_empty(), + "a second consumer in the same group re-consumes nothing" + ); +} + +/// `publish` + `subscribe`: each `group` is a distinct Kafka consumer group, and +/// Kafka delivers every record to every group — so each group reads every event +/// (fan-out). A fresh group reads from earliest. +#[tokio::test] +async fn bus_subscribe_fans_out_across_groups() { + let Some(brokers) = brokers() else { return }; + let ns = unique("ns"); + + let producer = KafkaBus::connect(&brokers, "producer", &ns) + .await + .expect("connect producer"); + let total = 4; + for i in 0..total { + producer + .publish_message( + Message::new("evt", MessageKind::Event, b"{}".to_vec()).with_id(format!("e{i}")), + ) + .await + .expect("publish event"); + } + + let expected: Vec = (0..total).map(|i| format!("e{i}")).collect(); + for group in ["projections", "audit"] { + let rec = Arc::new(Mutex::new(Vec::new())); + KafkaBus::connect(&brokers, group, &ns) + .await + .unwrap() + .with_fetch_timeout(Duration::from_secs(10)) + .subscribe( + recording_for("evt", MessageKind::Event, rec.clone()), + RunOptions::idempotent(), + ) + .await + .expect("subscriber drains"); + let mut ids = rec.lock().unwrap().clone(); + ids.sort(); + assert_eq!(ids, expected, "group {group} sees every event"); + } +} From 9cc8b40bd6aa4f2f8331d6410ac34513177b3754 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:27:40 -0500 Subject: [PATCH 16/39] feat(transport): add KnativeBus (Bus produce + manifest generation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the locked spec (Decision #6), KnativeBus implements only Bus and NOT BusConsumer — Knative is a GitOps/HTTP transport with no in-process consume loop: - produce: send/publish POST a binary-mode CloudEvent to a broker-ingress URL ({ingress_base}/{namespace}/{broker}); publish targets the service's own {source}-events broker, send targets a downstream {commands_broker}. A message without an id is rejected (CloudEvents mandates `id`). - consume = deploy-time artifacts: manifests(&plan, &subscriptions) renders role-based Broker + per-name Trigger YAML — own {source}-commands broker + command triggers if it handles commands, own {source}-events broker if publishes_events, and a Trigger per subscribed event on its producer's broker, with /cloudevent/ subscriber URIs. A .local(addr) builder switches subscribers to a kubefwd address. Adds the per-type /cloudevent/{type} route to cloud_events_router (Decision #7) and reqwest (optional, default-features off) under the http feature for the POST. Four tests: produce round-trips through a local cloud_events_router into dispatch_message; missing id rejected; manifests render brokers/triggers; pure-consumer (publishes_events=false) owns no broker and uses the local subscriber URI. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 3 +- src/microsvc/transport/knative.rs | 10 +- src/microsvc/transport/knative_bus.rs | 242 ++++++++++++++++++++++++++ src/microsvc/transport/mod.rs | 4 + tests/knative_cloudevents/main.rs | 111 +++++++++++- 5 files changed, 364 insertions(+), 6 deletions(-) create mode 100644 src/microsvc/transport/knative_bus.rs diff --git a/Cargo.toml b/Cargo.toml index fb2a226e2..2488aaf25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ path = "src/lib.rs" default = ["emitter", "bus"] emitter = ["dep:event-emitter-rs"] bus = [] -http = ["bus", "dep:axum", "dep:tokio"] +http = ["bus", "dep:axum", "dep:reqwest", "dep:tokio"] grpc = ["bus", "dep:tonic", "dep:prost", "dep:tokio"] postgres = ["dep:sqlx", "dep:tokio", "sqlx/postgres", "sqlx/runtime-tokio"] sqlite = ["dep:sqlx", "dep:tokio", "sqlx/runtime-tokio", "sqlx/sqlite"] @@ -45,6 +45,7 @@ base64 = "0.22.1" futures = { version = "0.3", optional = true } lapin = { version = "2", optional = true } rdkafka = { version = "0.36", features = ["cmake-build", "tokio"], optional = true } +reqwest = { version = "0.12", default-features = false, optional = true } bitcode = { version = "0.6.9", features = ["serde"] } event-emitter-rs = { version = "0.1.4", optional = true } serde = { version = "1.0.210", features = ["derive"] } diff --git a/src/microsvc/transport/knative.rs b/src/microsvc/transport/knative.rs index 599cd1b4b..43d76de8f 100644 --- a/src/microsvc/transport/knative.rs +++ b/src/microsvc/transport/knative.rs @@ -34,13 +34,17 @@ use crate::microsvc::{Message, MessageKind, Service, SubscriptionPlan}; const STRUCTURED_CONTENT_TYPE: &str = "application/cloudevents+json"; -/// Build an axum router exposing a CloudEvents ingress at `POST /`. +/// Build an axum router exposing a CloudEvents ingress at `POST /` and the +/// per-type route `POST /cloudevent/:type`. /// -/// Compose it with other routes or serve it directly; Knative Triggers point a -/// subscriber `ref` at this endpoint. +/// Both dispatch by the parsed CloudEvent `type` (the path segment is for routing +/// alignment only), so a Knative Trigger can target either a single shared `ref` +/// (`/`) or the per-type subscriber URI [`KnativeBus`](super::KnativeBus) emits +/// (`/cloudevent/`). Compose it with other routes or serve it directly. pub fn cloud_events_router(service: Arc>) -> Router { Router::new() .route("/", axum::routing::post(ingress_handler)) + .route("/cloudevent/:type", axum::routing::post(ingress_handler)) .with_state(service) } diff --git a/src/microsvc/transport/knative_bus.rs b/src/microsvc/transport/knative_bus.rs new file mode 100644 index 000000000..0e8572715 --- /dev/null +++ b/src/microsvc/transport/knative_bus.rs @@ -0,0 +1,242 @@ +//! Knative Eventing [`Bus`] — produce side + manifest generation. +//! +//! Knative is a GitOps/K8s transport: the platform delivers events over HTTP to +//! per-handler routes wired by `Trigger` manifests, so there is no in-process +//! consume loop. Per the locked spec (Decision #6), `KnativeBus` implements only +//! [`Bus`] (produce) and does **not** implement +//! [`BusConsumer`](super::BusConsumer). Consuming is two deploy-time artifacts: +//! +//! - [`KnativeBus::manifests`] — the role-based `Broker` + per-name `Trigger` +//! YAML for a service (and a local/kubefwd variant); +//! - mounting [`cloud_events_router`](super::cloud_events_router) so the Triggers' +//! `/cloudevent/` subscriber URIs reach `Service::dispatch_message`. +//! +//! Producing POSTs a binary-mode CloudEvent to a broker-ingress URL: +//! `publish` → the service's own `{source}-events` broker; `send` → a target +//! model's `{target}-commands` broker. Owning a broker is only required for what +//! you *receive*; publishing just needs the ingress URL. +//! +//! Requires the `http` feature. + +use std::time::Duration; + +use super::{Bus, TransportError}; +use crate::microsvc::{Message, MessageKind, SubscriptionPlan}; + +const SEND_TIMEOUT: Duration = Duration::from_secs(10); + +fn kind_str(kind: MessageKind) -> &'static str { + match kind { + MessageKind::Command => "command", + MessageKind::Event => "event", + } +} + +/// Knative Eventing [`Bus`] (produce) + manifest generator. +#[derive(Clone)] +pub struct KnativeBus { + client: reqwest::Client, + /// Broker-ingress base, e.g. `http://broker-ingress.knative-eventing.svc.cluster.local`. + ingress_base: String, + /// Kubernetes namespace (broker-ingress path segment + manifest `namespace`). + namespace: String, + /// This service's name — the CloudEvent `source` and the manifest subject. + source: String, + /// Target commands broker for `send` (a downstream model's `*-commands`). + commands_broker: String, + /// This service's own events broker for `publish` (`{source}-events`). + events_broker: String, + /// Whether `manifests` should emit this service's `{source}-events` broker. + publishes_events: bool, + /// When set, `manifests` points Trigger subscribers at this local address + /// (kubefwd dev flow) instead of the in-cluster service `ref`. + local: Option, +} + +impl KnativeBus { + /// Build a bus. `ingress_base` is the Knative broker-ingress base URL, + /// `source` is this service's name, and `commands_broker`/`events_broker` are + /// the brokers `send`/`publish` POST to. + pub fn new( + ingress_base: impl Into, + namespace: impl Into, + source: impl Into, + commands_broker: impl Into, + events_broker: impl Into, + ) -> Self { + Self { + client: reqwest::Client::new(), + ingress_base: ingress_base.into(), + namespace: namespace.into(), + source: source.into(), + commands_broker: commands_broker.into(), + events_broker: events_broker.into(), + publishes_events: true, + local: None, + } + } + + /// Set whether [`manifests`](Self::manifests) emits the `{source}-events` + /// broker (true for model/ingestor services that publish; false for pure + /// consumers like projections). Default: true. + pub fn publishes_events(mut self, publishes: bool) -> Self { + self.publishes_events = publishes; + self + } + + /// Point generated Trigger subscribers at `addr` (e.g. `127.0.0.1:8080`) via + /// a local URI instead of the in-cluster service `ref` — the kubefwd dev flow. + pub fn local(mut self, addr: impl Into) -> Self { + self.local = Some(addr.into()); + self + } + + /// Join `ingress_base` / `namespace` / `broker`, skipping empty segments. + fn ingress_url(&self, broker: &str) -> String { + let mut url = self.ingress_base.trim_end_matches('/').to_string(); + for segment in [self.namespace.as_str(), broker] { + if !segment.is_empty() { + url.push('/'); + url.push_str(segment); + } + } + url + } + + async fn post_cloud_event( + &self, + broker: &str, + message: &Message, + ) -> Result<(), TransportError> { + let Some(id) = message.id() else { + return Err(TransportError::permanent( + "knative produce requires a message id (CloudEvents `id` is mandatory)", + )); + }; + let mut request = self + .client + .post(self.ingress_url(broker)) + .timeout(SEND_TIMEOUT) + .header("ce-specversion", "1.0") + .header("ce-id", id) + .header("ce-type", message.name()) + .header("ce-source", self.source.as_str()) + .header("ce-sourcedkind", kind_str(message.kind)) + .header("content-type", message.content_type.as_str()); + for (key, value) in &message.metadata { + request = request.header(format!("ce-{key}"), value); + } + let response = request + .body(message.payload.clone()) + .send() + .await + .map_err(|err| TransportError::retryable(format!("knative POST: {err}")))?; + if !response.status().is_success() { + return Err(TransportError::retryable(format!( + "knative broker-ingress returned {}", + response.status() + ))); + } + Ok(()) + } + + /// Render the role-based Knative manifests for `plan`: the `{source}-commands` + /// broker + a Trigger per handled command (if any), the `{source}-events` + /// broker (if [`publishes_events`](Self::publishes_events)), and a Trigger per + /// subscribed event on its producer's broker. `subscriptions` maps each + /// subscribed event name to the producing service's events broker. Subscriber + /// URIs are `/cloudevent/` (matching [`cloud_events_router`]). + /// + /// [`cloud_events_router`]: super::cloud_events_router + pub fn manifests(&self, plan: &SubscriptionPlan, subscriptions: &[(&str, &str)]) -> String { + let mut out = String::new(); + if !plan.commands.is_empty() { + // The broker a service *owns* for commands it handles is + // `{source}-commands` — distinct from `commands_broker` (the + // downstream target `send` produces to). + let own_commands = format!("{}-commands", self.source); + out.push_str(&self.broker_yaml(&own_commands)); + for command in &plan.commands { + out.push_str(&self.trigger_yaml(&own_commands, command)); + } + } + if self.publishes_events { + out.push_str(&self.broker_yaml(&self.events_broker)); + } + for event in &plan.events { + let broker = subscriptions + .iter() + .find(|(name, _)| *name == event.as_str()) + .map(|(_, broker)| *broker) + .unwrap_or("UNMAPPED-events"); + out.push_str(&self.trigger_yaml(broker, event)); + } + out + } + + fn broker_yaml(&self, name: &str) -> String { + format!( + "apiVersion: eventing.knative.dev/v1\n\ + kind: Broker\n\ + metadata:\n\ + \x20 name: {name}\n\ + \x20 namespace: {ns}\n\ + ---\n", + ns = self.namespace, + ) + } + + fn trigger_yaml(&self, broker: &str, event: &str) -> String { + let trigger_name = format!("{}-{}", self.source, event.replace('.', "-")); + let subscriber = match &self.local { + Some(addr) => format!( + "\x20 subscriber:\n\ + \x20 uri: http://{addr}/cloudevent/{event}\n" + ), + None => format!( + "\x20 subscriber:\n\ + \x20 ref:\n\ + \x20 apiVersion: serving.knative.dev/v1\n\ + \x20 kind: Service\n\ + \x20 name: {source}\n\ + \x20 uri: /cloudevent/{event}\n", + source = self.source, + ), + }; + format!( + "apiVersion: eventing.knative.dev/v1\n\ + kind: Trigger\n\ + metadata:\n\ + \x20 name: {trigger_name}\n\ + \x20 namespace: {ns}\n\ + spec:\n\ + \x20 broker: {broker}\n\ + \x20 filter:\n\ + \x20 attributes:\n\ + \x20 type: {event}\n\ + {subscriber}\ + ---\n", + ns = self.namespace, + ) + } +} + +impl Bus for KnativeBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.send_message(Message::new(name, MessageKind::Command, payload)) + .await + } + + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.publish_message(Message::new(name, MessageKind::Event, payload)) + .await + } + + async fn send_message(&self, message: Message) -> Result<(), TransportError> { + self.post_cloud_event(&self.commands_broker, &message).await + } + + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.post_cloud_event(&self.events_broker, &message).await + } +} diff --git a/src/microsvc/transport/mod.rs b/src/microsvc/transport/mod.rs index 6f0815a04..320e316df 100644 --- a/src/microsvc/transport/mod.rs +++ b/src/microsvc/transport/mod.rs @@ -92,6 +92,8 @@ mod kafka; mod kafka_bus; #[cfg(feature = "http")] mod knative; +#[cfg(feature = "http")] +mod knative_bus; #[cfg(feature = "nats")] mod nats; #[cfg(feature = "nats")] @@ -116,6 +118,8 @@ pub use kafka::{KafkaPublisher, KafkaReceived, KafkaSource}; pub use kafka_bus::KafkaBus; #[cfg(feature = "http")] pub use knative::{cloud_events_router, knative_triggers}; +#[cfg(feature = "http")] +pub use knative_bus::KnativeBus; #[cfg(feature = "nats")] pub use nats::{NatsJetStreamSource, NatsPublisher, NatsReceived}; #[cfg(feature = "nats")] diff --git a/tests/knative_cloudevents/main.rs b/tests/knative_cloudevents/main.rs index 2d2ee2203..cf4884f30 100644 --- a/tests/knative_cloudevents/main.rs +++ b/tests/knative_cloudevents/main.rs @@ -8,8 +8,8 @@ use std::sync::{Arc, Mutex}; use serde_json::json; -use sourced_rust::microsvc::transport::cloud_events_router; -use sourced_rust::microsvc::{HandlerError, Service}; +use sourced_rust::microsvc::transport::{cloud_events_router, Bus, KnativeBus}; +use sourced_rust::microsvc::{HandlerError, Message, MessageKind, Service, SubscriptionPlan}; async fn spawn_server() -> (String, Arc>>) { let handled = Arc::new(Mutex::new(Vec::::new())); @@ -139,3 +139,110 @@ async fn missing_id_returns_400() { .unwrap(); assert_eq!(resp.status(), 400); } + +// ---- KnativeBus: produce (POST CloudEvent) + manifest generation ---- + +/// `KnativeBus::publish` POSTs a binary CloudEvent to the broker-ingress URL; +/// pointed at the local router it round-trips into `Service::dispatch_message`. +#[tokio::test] +async fn knative_bus_publish_round_trips_through_router() { + let (url, handled) = spawn_server().await; + // ingress_base = the server (no broker/namespace) so the POST hits `POST /`. + let bus = KnativeBus::new(url.trim_end_matches('/'), "", "orders-svc", "", ""); + bus.publish_message( + Message::new( + "order.created", + MessageKind::Event, + br#"{"order":"o9"}"#.to_vec(), + ) + .with_id("evt-9"), + ) + .await + .expect("publish round-trips"); + // publish awaits the HTTP response, which the router returns only after + // dispatch, so the handler has already recorded by now. + assert_eq!(handled.lock().unwrap().clone(), vec!["evt-9".to_string()]); +} + +/// A produce with no message id is rejected before any POST (CloudEvents +/// mandates `id`). +#[tokio::test] +async fn knative_bus_publish_without_id_is_rejected() { + let bus = KnativeBus::new("http://127.0.0.1:1", "", "orders-svc", "", ""); + let err = bus + .publish_message(Message::new( + "order.created", + MessageKind::Event, + b"{}".to_vec(), + )) + .await + .expect_err("missing id is rejected"); + assert!( + err.to_string().contains("id"), + "error mentions the missing id: {err}" + ); +} + +/// `manifests` renders the role-based Brokers + per-name Triggers a service's +/// chart needs, with `/cloudevent/` subscriber URIs. +#[test] +fn knative_bus_manifests_render_brokers_and_triggers() { + let bus = KnativeBus::new( + "http://broker-ingress.knative-eventing.svc.cluster.local", + "game", + "model-svc", + "lobby-svc-commands", + "model-svc-events", + ); + let plan = SubscriptionPlan { + commands: vec!["place.bet".to_string()], + events: vec!["seat.reserved".to_string()], + }; + let yaml = bus.manifests(&plan, &[("seat.reserved", "lobby-svc-events")]); + + // Own commands broker + a Trigger for the handled command. + assert!(yaml.contains("kind: Broker")); + assert!(yaml.contains("name: model-svc-commands")); + assert!(yaml.contains("type: place.bet")); + // Own events broker (this service publishes). + assert!(yaml.contains("name: model-svc-events")); + // Subscribed event → Trigger on the *producer's* broker, per-type subscriber. + assert!(yaml.contains("broker: lobby-svc-events")); + assert!(yaml.contains("type: seat.reserved")); + assert!(yaml.contains("uri: /cloudevent/seat.reserved")); + assert!(yaml.contains("namespace: game")); +} + +/// A pure consumer (`publishes_events(false)`) creates no events broker; the +/// local flag points subscribers at a kubefwd address. +#[test] +fn knative_bus_manifests_pure_consumer_local_variant() { + let bus = KnativeBus::new( + "http://ingress", + "game", + "projection-svc", + "", + "projection-svc-events", + ) + .publishes_events(false) + .local("127.0.0.1:8080"); + let plan = SubscriptionPlan { + commands: vec![], + events: vec!["seat.reserved".to_string()], + }; + let yaml = bus.manifests(&plan, &[("seat.reserved", "lobby-svc-events")]); + + assert!( + !yaml.contains("projection-svc-events"), + "pure consumer owns no broker" + ); + assert!( + !yaml.contains("kind: Broker"), + "pure consumer creates no broker" + ); + assert!(yaml.contains("kind: Trigger")); + assert!( + yaml.contains("uri: http://127.0.0.1:8080/cloudevent/seat.reserved"), + "local flag points the subscriber at the kubefwd address" + ); +} From e19fb3adcbec1f2c1786cc13ecf8a36edf4b2c8f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:33:19 -0500 Subject: [PATCH 17/39] docs(transport): document the bus facade + transport-swap example Add a "Bus facade" section to docs/async-transports.md: the Bus / BusConsumer surface, a transport-swap example (same service + handlers, one constructor line changes), the per-transport competing-vs-fan-out topology table, and the Knative Bus-only + manifests note. Refresh Status. Implements [[tasks/build-transport-bus-facade]] Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/async-transports.md | 77 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/async-transports.md b/docs/async-transports.md index 00a612910..4631f8ae9 100644 --- a/docs/async-transports.md +++ b/docs/async-transports.md @@ -93,6 +93,66 @@ Retry/backoff/dead-lettering ownership differs: with Knative it is **platform-managed** (Delivery/Trigger config); with direct transports the adapter and this crate own it via the `FailurePolicy` and the outbox lease. +## Bus facade: `send`/`listen` + `publish`/`subscribe` + +The adapters above are the low-level boundary. The **bus facade** is the +ergonomic surface on top: a produce trait [`Bus`] (`send` a command, `publish` an +event) and a consume trait [`BusConsumer`] (`listen` for commands, `subscribe` to +events), implemented by a per-transport `*Bus` type. `listen`/`subscribe` derive +the message names from the service's registered handlers +(`command_names()`/`event_names()`), build the transport's source with the right +topology, and run it through the shared `run_source` — handler code and +`dispatch_message` never change. + +The app surface is identical across transports; only the constructor changes: + +```rust +use std::sync::Arc; +use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; + +// Built once — handlers are transport-agnostic. +let service = Arc::new(build_service()); + +// Dev/test: in-memory. +let bus = InMemoryBus::new(); +bus.send("place.bet", payload).await?; // point-to-point command (1:1) +bus.publish("seat.reserved", payload).await?; // fan-out event (1:N) +bus.listen(service.clone(), RunOptions::idempotent()).await?; // competing +bus.subscribe(service.clone(), RunOptions::idempotent()).await?; // fan-out + +// Production: swap the one constructor line — send/listen/publish/subscribe +// and the handlers are unchanged. +// let bus = NatsBus::connect("nats://localhost:4222", "orders", "app").await?; +// let bus = PostgresBus::new(pool, "orders"); +// let bus = RabbitBus::connect("amqp://localhost:5672/%2f", "orders", "app").await?; +// let bus = KafkaBus::connect("localhost:9092", "orders", "app").await?; +``` + +Point-to-point vs fan-out is consistently a **consumer-group/identity** choice in +each transport's native topology — same `group` competes, different `group`s +fan out: + +| `*Bus` | Feature | `send` / `listen` (competing) | `publish` / `subscribe` (fan-out) | +| --- | --- | --- | --- | +| `InMemoryBus` | (always) | named queue, popped once | retained log + per-subscriber cursor | +| `NatsBus` | `nats` | shared durable `{group}_cmd` on the stream | durable `{group}_evt` per group | +| `PostgresBus` | `postgres` | `bus_queue`, `FOR UPDATE SKIP LOCKED` | `bus_log` + `bus_offset` per `group` (Kafka-style) | +| `RabbitBus` | `rabbitmq` | default exchange → durable queue `{ns}.cmd.{name}` | topic exchange → queue `{ns}.evt.{group}` per group | +| `KafkaBus` | `kafka` | shared consumer group `{ns}.{group}.cmd` | consumer group per service `{ns}.{group}.evt` | +| `KnativeBus` | `http` | POST CloudEvent → `{target}-commands` broker-ingress | POST → own `{source}-events` broker; consume via generated Triggers | + +`KnativeBus` implements only [`Bus`] (produce → broker-ingress POST). It has no +in-process consume loop: `KnativeBus::manifests(&plan, &subscriptions)` renders +the role-based `Broker` + per-name `Trigger` YAML (subscriber URIs +`/cloudevent/`, with a `.local(addr)` kubefwd variant), and the service +mounts `cloud_events_router` so those Triggers reach `dispatch_message`. + +`PostgresBus` uses the claim-lease work queue (not `sqlxmq`) for the same reason +the low-level adapter does — sqlxmq's always-on push runner doesn't compose with +the uniform drain-to-idle `run_source` model the facade shares; its `bus_log` + +`bus_offset` fan-out gives single-DB transactional effectively-once (the offset +advances with the effects). See `specs/transport-bus-facade`. + ## Testing The reusable conformance harness (`tests/transport_conformance/`) proves the @@ -111,14 +171,19 @@ AMQP_URL=amqp://guest:guest@localhost:5672/%2f \ KAFKA_BROKERS=localhost:9092 cargo test --test kafka_transport --features kafka ``` -Each broker has a matching GitHub Actions job (reusable -`.github/workflows/integration-*.yaml`) that runs on PRs and on push to `main`. +Each transport's integration binary also covers its `*Bus`: a competing-consumer +case (one delivery across a shared group) and a fan-out case (every group sees +every event), verified against the real broker. Each broker has a matching GitHub +Actions job (reusable `.github/workflows/integration-*.yaml`) that runs on PRs and +on push to `main`. ## Status Implemented and verified: the core contracts, the source runner, the publisher / outbox dispatcher, the conformance harness, the Postgres / NATS / RabbitMQ / -Kafka adapters, and the Knative ingress. Still open: migrating the in-repo -examples to showcase these APIs and removing the legacy synchronous bus paths -(a breaking change), and a long-running poll/notify consumer daemon for the -Postgres source. See `tasks/transport-docs-examples-cutover`. +Kafka adapters, the Knative ingress, and the **bus facade** (`Bus` + +`BusConsumer` with `InMemoryBus` / `NatsBus` / `PostgresBus` / `RabbitBus` / +`KafkaBus` / `KnativeBus`, each with real-broker competing-vs-fan-out tests). +Still open: migrating the in-repo examples to showcase these APIs and removing the +legacy synchronous bus paths (a breaking change). See +`tasks/transport-docs-examples-cutover`. From f528994f77716b12da755f10f9d3513d1bbf762d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 00:55:15 -0500 Subject: [PATCH 18/39] ci: install libcurl dev headers for the rdkafka build The kafka, postgres (--all-features), and coverage (--all-features) jobs compile rdkafka-sys, whose bundled librdkafka cmake build enables curl (OAUTHBEARER OIDC) when it finds the runner's libcurl runtime, then fails with `curl/curl.h: No such file or directory` because the dev headers aren't installed. Install libcurl4-openssl-dev in those three jobs. nats/rabbitmq use narrow feature sets, build no rdkafka, and were green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/integration-kafka.yaml | 7 ++++++- .github/workflows/integration-postgres.yaml | 5 +++++ .github/workflows/rust.yml | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-kafka.yaml b/.github/workflows/integration-kafka.yaml index a2c05efbd..0f5e763e0 100644 --- a/.github/workflows/integration-kafka.yaml +++ b/.github/workflows/integration-kafka.yaml @@ -42,6 +42,11 @@ jobs: - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 with: toolchain: stable - # ubuntu-latest ships cmake + gcc, which rdkafka's cmake-build needs. + # ubuntu-latest ships cmake + gcc, which rdkafka's cmake-build needs. It + # also has libcurl's runtime but not its headers; librdkafka's bundled + # build enables curl (OAUTHBEARER OIDC) when it finds the lib and then + # needs curl/curl.h, so install the dev package. + - name: Install librdkafka build dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Run Kafka transport integration tests run: cargo test --test kafka_transport --features kafka --verbose diff --git a/.github/workflows/integration-postgres.yaml b/.github/workflows/integration-postgres.yaml index 63bc0d7e6..343ea6daa 100644 --- a/.github/workflows/integration-postgres.yaml +++ b/.github/workflows/integration-postgres.yaml @@ -33,6 +33,11 @@ jobs: - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 with: toolchain: stable + # These tests run with --all-features, which compiles rdkafka; librdkafka's + # bundled cmake build needs curl/curl.h (the runner has libcurl's runtime + # but not its headers). + - name: Install librdkafka build dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Run Postgres integration tests run: | cargo test --test postgres_repository --all-features --verbose diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b2bed1348..36e03f3cc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,6 +20,10 @@ jobs: with: components: llvm-tools-preview - uses: taiki-e/install-action@cargo-llvm-cov + # --all-features compiles rdkafka; librdkafka's bundled cmake build needs + # curl/curl.h (the runner has libcurl's runtime but not its headers). + - name: Install librdkafka build dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Generate coverage report run: cargo llvm-cov --all-features --verbose --lcov --output-path lcov.info - name: Upload coverage artifact From f91c9fc55015a757ddeed8e5ddf01ed59028b596 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 01:01:05 -0500 Subject: [PATCH 19/39] docs(read-model): clarify ReadModelCommitOutcome is an intentional stub Address the CodeRabbit review on PR #47. was_applied() is hardcoded true not by accident: the read_model_processed_messages dedupe table and the skipped_duplicate outcome were deliberately removed (see specs/consumer-inbox-design.md, 2026-05-28) because coupling delivery dedupe to the read-model projection contract was the wrong boundary. Replay safety is now a projection convention (idempotent handlers + per-row ExpectedVersion OCC); a first-class replay barrier returns with the consumer inbox as a CommitBatch participant. Document this on the type so the always-true was_applied() isn't misread as a lost signal. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/read_model/session.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/read_model/session.rs b/src/read_model/session.rs index a7829760c..06691fca9 100644 --- a/src/read_model/session.rs +++ b/src/read_model/session.rs @@ -56,14 +56,28 @@ impl Default for ReadModelAdapterCapabilities { } /// Result of applying a standalone read-model write plan. +/// +/// This is intentionally a stub: it carries no skipped/replay state and +/// [`was_applied`](Self::was_applied) is always `true`. The earlier +/// `read_model_processed_messages` dedupe table and `skipped_duplicate` outcome +/// were **deliberately removed** (see `specs/consumer-inbox-design.md`, decision +/// 2026-05-28) because coupling delivery-level dedupe to the read-model +/// projection contract was the wrong boundary. Replay safety is now a projection +/// convention — handlers make their writes idempotent so a redelivered event +/// re-converges (plus per-row `ExpectedVersion` optimistic concurrency). A +/// first-class replay barrier returns with the consumer inbox (an operational +/// `consumer_inbox` table committed as a `CommitBatch` participant), tracked +/// under `tasks/build-transport-bus-facade`; the variant set will grow then. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReadModelCommitOutcome; impl ReadModelCommitOutcome { + /// The write plan was applied. Currently the only outcome (see the type docs). pub fn applied() -> Self { Self } + /// Always `true` today — see the type docs for why there is no skipped variant. pub fn was_applied(&self) -> bool { true } From 4ef3a6c823ee965784e6d6388fd6c55e544a333e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 01:08:11 -0500 Subject: [PATCH 20/39] fix(transport): address CodeRabbit review on PR #47 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three valid review findings: - outbox_source: reject zero lease / batch_size in the builder. A zero batch makes recv() return Ok(None) forever; a zero lease makes claimed rows immediately re-claimable. Both are silent misconfigurations on a public builder — assert up front. (+ #[should_panic] tests) - rabbitmq: preserve Message.content_type across the AMQP round-trip. message_properties now sets AMQP content_type, and from_delivery reads it back instead of letting Message::new hardcode application/json. Also fixes the same loss for RabbitBus (shares both paths). (+ round-trip test asserting a non-JSON content type survives) - knative: sanitize generated Trigger names to RFC 1123 (lowercase, alphanumeric/'-', no leading/trailing '-', <=63 chars) via a shared sanitize_k8s_name helper, applied in both knative_triggers and KnativeBus::trigger_yaml. CloudEvent types can contain dots/uppercase that are invalid in k8s resource names. (+ unit tests) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/transport/knative.rs | 57 ++++++++++++++++++++++++- src/microsvc/transport/knative_bus.rs | 3 +- src/microsvc/transport/outbox_source.rs | 30 +++++++++++++ src/microsvc/transport/rabbitmq.rs | 9 +++- tests/rabbitmq_transport/main.rs | 20 +++++---- 5 files changed, 108 insertions(+), 11 deletions(-) diff --git a/src/microsvc/transport/knative.rs b/src/microsvc/transport/knative.rs index 43d76de8f..d06a2473b 100644 --- a/src/microsvc/transport/knative.rs +++ b/src/microsvc/transport/knative.rs @@ -188,13 +188,38 @@ fn parse_structured(body: &Bytes) -> Result { }) } +/// Sanitize a string into an RFC 1123 DNS label usable as a Kubernetes resource +/// name: lowercase, ASCII-alphanumeric or `-`, no leading/trailing `-`, ≤63 +/// chars. Used for generated `Trigger` names, whose CloudEvent-type segment can +/// contain dots/uppercase/other characters that are invalid in k8s names. +pub(super) fn sanitize_k8s_name(name: &str) -> String { + let mapped: String = name + .chars() + .map(|c| { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphanumeric() { + c + } else { + '-' + } + }) + .collect(); + let capped: String = mapped.trim_matches('-').chars().take(63).collect(); + let trimmed = capped.trim_end_matches('-'); + if trimmed.is_empty() { + "x".to_string() + } else { + trimmed.to_string() + } +} + /// Render Knative `Trigger` YAML for each event a service subscribes to, derived /// from its [`SubscriptionPlan`]. Each Trigger filters on the CloudEvent `type` /// and routes to `subscriber_service` on `broker`. pub fn knative_triggers(plan: &SubscriptionPlan, broker: &str, subscriber_service: &str) -> String { let mut out = String::new(); for event in &plan.events { - let trigger_name = format!("{subscriber_service}-{}", event.replace('.', "-")); + let trigger_name = sanitize_k8s_name(&format!("{subscriber_service}-{event}")); out.push_str(&format!( "apiVersion: eventing.knative.dev/v1\n\ kind: Trigger\n\ @@ -287,4 +312,34 @@ mod tests { assert!(yaml.contains("name: checkout-projection-seat-reserved")); assert!(yaml.contains("broker: default")); } + + #[test] + fn sanitize_k8s_name_enforces_rfc1123() { + // Valid names pass through unchanged. + assert_eq!( + sanitize_k8s_name("checkout-projection-seat-reserved"), + "checkout-projection-seat-reserved" + ); + // Dots, uppercase, and other characters become '-' and lowercase. + assert_eq!(sanitize_k8s_name("Order.Created!"), "order-created"); + // No leading/trailing dashes, capped at 63 chars. + let long = "a".repeat(80); + let out = sanitize_k8s_name(&format!(".{long}.")); + assert_eq!(out.len(), 63); + assert!(!out.starts_with('-') && !out.ends_with('-')); + // All-invalid degrades to a safe placeholder. + assert_eq!(sanitize_k8s_name("..."), "x"); + } + + #[test] + fn trigger_name_is_sanitized_for_messy_event_types() { + let plan = SubscriptionPlan { + commands: vec![], + events: vec!["Order.Created".to_string()], + }; + let yaml = knative_triggers(&plan, "default", "checkout-projection"); + // The CloudEvent type filter keeps the raw type; the resource name is sanitized. + assert!(yaml.contains("type: Order.Created")); + assert!(yaml.contains("name: checkout-projection-order-created")); + } } diff --git a/src/microsvc/transport/knative_bus.rs b/src/microsvc/transport/knative_bus.rs index 0e8572715..07a3c764f 100644 --- a/src/microsvc/transport/knative_bus.rs +++ b/src/microsvc/transport/knative_bus.rs @@ -20,6 +20,7 @@ use std::time::Duration; +use super::knative::sanitize_k8s_name; use super::{Bus, TransportError}; use crate::microsvc::{Message, MessageKind, SubscriptionPlan}; @@ -187,7 +188,7 @@ impl KnativeBus { } fn trigger_yaml(&self, broker: &str, event: &str) -> String { - let trigger_name = format!("{}-{}", self.source, event.replace('.', "-")); + let trigger_name = sanitize_k8s_name(&format!("{}-{}", self.source, event)); let subscriber = match &self.local { Some(addr) => format!( "\x20 subscriber:\n\ diff --git a/src/microsvc/transport/outbox_source.rs b/src/microsvc/transport/outbox_source.rs index 22b33f54d..1cb4a40eb 100644 --- a/src/microsvc/transport/outbox_source.rs +++ b/src/microsvc/transport/outbox_source.rs @@ -59,13 +59,29 @@ where } /// Set the claim lease / visibility timeout. + /// + /// # Panics + /// Panics if `lease` is zero: a zero lease makes a claimed row immediately + /// re-claimable by competing workers, defeating the lease. pub fn with_lease(mut self, lease: Duration) -> Self { + assert!( + !lease.is_zero(), + "OutboxSource lease must be greater than zero" + ); self.lease = lease; self } /// Set how many rows are claimed per refill. + /// + /// # Panics + /// Panics if `batch_size` is zero: a zero batch claims nothing, so `recv` + /// would return `Ok(None)` forever even when rows are pending. pub fn with_batch_size(mut self, batch_size: usize) -> Self { + assert!( + batch_size > 0, + "OutboxSource batch_size must be greater than zero" + ); self.batch_size = batch_size; self } @@ -216,6 +232,20 @@ mod tests { OutboxSource::new(Arc::new(repo.outbox_store()), "pg-transport", 3) } + #[test] + #[should_panic(expected = "lease must be greater than zero")] + fn with_lease_zero_panics() { + let repo = HashMapRepository::new(); + let _ = source(&repo).with_lease(Duration::ZERO); + } + + #[test] + #[should_panic(expected = "batch_size must be greater than zero")] + fn with_batch_size_zero_panics() { + let repo = HashMapRepository::new(); + let _ = source(&repo).with_batch_size(0); + } + #[test] fn recv_yields_claimed_rows_then_drains_to_none() { let repo = HashMapRepository::new(); diff --git a/src/microsvc/transport/rabbitmq.rs b/src/microsvc/transport/rabbitmq.rs index d2df99029..32183928a 100644 --- a/src/microsvc/transport/rabbitmq.rs +++ b/src/microsvc/transport/rabbitmq.rs @@ -72,7 +72,9 @@ pub(super) fn message_properties(message: &Message) -> BasicProperties { AMQPValue::LongString(value.as_str().into()), ); } - let mut properties = BasicProperties::default().with_headers(headers); + let mut properties = BasicProperties::default() + .with_headers(headers) + .with_content_type(ShortString::from(message.content_type.as_str())); if let Some(id) = message.id() { properties = properties.with_message_id(ShortString::from(id)); } @@ -189,6 +191,11 @@ impl RabbitReceived { let mut message = Message::new(name, kind, payload); message.id = id; message.metadata = metadata; + // Preserve the publisher's content type instead of the Message::new + // default, so non-JSON payloads survive the round-trip. + if let Some(content_type) = delivery.properties.content_type().as_ref() { + message.content_type = content_type.to_string(); + } Self { delivery, message } } } diff --git a/tests/rabbitmq_transport/main.rs b/tests/rabbitmq_transport/main.rs index 1947ffa36..034bff57e 100644 --- a/tests/rabbitmq_transport/main.rs +++ b/tests/rabbitmq_transport/main.rs @@ -116,14 +116,13 @@ async fn message_id_and_metadata_survive_the_round_trip() { let publisher = RabbitPublisher::connect(&url) .await .expect("connect publisher"); - publisher - .publish( - Message::new(&queue, MessageKind::Event, br#"{"k":"v"}"#.to_vec()) - .with_id("evt-1") - .with_metadata("correlation_id", "corr-9"), - ) - .await - .expect("publish"); + // Use a non-default content type to prove it survives the AMQP round-trip + // (Message::new defaults to application/json). + let mut message = Message::new(&queue, MessageKind::Event, br#"{"k":"v"}"#.to_vec()) + .with_id("evt-1") + .with_metadata("correlation_id", "corr-9"); + message.content_type = "application/cloudevents+json".to_string(); + publisher.publish(message).await.expect("publish"); let observed = Arc::new(Mutex::new(None)); let o = observed.clone(); @@ -136,6 +135,7 @@ async fn message_id_and_metadata_survive_the_round_trip() { m.id().map(str::to_string), m.correlation_id().map(str::to_string), m.payload().to_vec(), + m.content_type.clone(), )); Ok(json!({})) }), @@ -148,6 +148,10 @@ async fn message_id_and_metadata_survive_the_round_trip() { assert_eq!(got.0.as_deref(), Some("evt-1")); assert_eq!(got.1.as_deref(), Some("corr-9")); assert_eq!(got.2, br#"{"k":"v"}"#.to_vec()); + assert_eq!( + got.3, "application/cloudevents+json", + "content_type round-trips" + ); } // ---- RabbitBus: send/listen (default exchange) + publish/subscribe (topic exchange) ---- From 6213cc82e3b938404b27c3f12e8d7f56ce89d1ad Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 01:41:20 -0500 Subject: [PATCH 21/39] feat(inbox): add InboxReceipt/InboxOutcome + CommitBatch participant field First slices of the consumer inbox (tasks/model-consumer-inbox-across- persistence-implementations, per specs/consumer-inbox-design): - New src/repository/inbox.rs: InboxReceipt { consumer, message_id, processed_at } and InboxOutcome { Processed, Duplicate } (Duplicate is success, never an error). Relocates the naming/semantics of the removed read_model_processed_messages into a first-class, non-read-model type. - Add inbox_receipts: Vec to both CommitBatch (sync) and AsyncCommitBatch (async), defaulted empty in new()/empty() and at every literal construction site. Trait signatures unchanged; empty everywhere so behavior is unchanged (257 lib tests green). The receipt is a commit-batch participant so it commits atomically with handler effects (the effect fence); storage writes + runner wiring follow in subsequent slices. Implements [[model-consumer-inbox-across-persistence-implementa]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/commit_builder/mod.rs | 6 ++ src/lib.rs | 4 +- src/repository/async_repository.rs | 5 ++ src/repository/batch.rs | 5 ++ src/repository/inbox.rs | 97 ++++++++++++++++++++++++++++++ src/repository/mod.rs | 2 + src/snapshot/repository.rs | 5 ++ 7 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/repository/inbox.rs diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index be6a20c11..21febbb89 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -172,6 +172,7 @@ impl<'a, R> SyncCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) } @@ -197,6 +198,7 @@ impl<'a, R> SyncCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) } @@ -213,6 +215,7 @@ impl<'a, R> SyncCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) } @@ -291,6 +294,7 @@ impl<'a, R> SyncStagedCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) } @@ -445,6 +449,7 @@ impl<'a, R> AsyncCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) .await } @@ -512,6 +517,7 @@ impl<'a, R> AsyncStagedCommitBuilder<'a, R> { outbox_messages: self.outbox_messages, read_model_plans: self.read_model_plans, snapshots: Vec::new(), + inbox_receipts: Vec::new(), }) .await } diff --git a/src/lib.rs b/src/lib.rs index fca4b85a6..16cb1d616 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,8 +42,8 @@ pub use repository::{ AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, AsyncTransactionalCommit, Commit, CommitBatch, Get, GetMany, GetOne, - Gettable, PreparedEventAppend, Repository, RepositoryError, SnapshotWrite, StreamIdentity, - TransactionalCommit, + Gettable, InboxOutcome, InboxReceipt, PreparedEventAppend, Repository, RepositoryError, + SnapshotWrite, StreamIdentity, TransactionalCommit, }; // Re-export aggregate types at crate root for convenience diff --git a/src/repository/async_repository.rs b/src/repository/async_repository.rs index 9a6240588..c6c5c98be 100644 --- a/src/repository/async_repository.rs +++ b/src/repository/async_repository.rs @@ -8,6 +8,7 @@ use crate::read_model::{ }; use crate::snapshot::SnapshotRecord; +use super::inbox::InboxReceipt; use super::{RepositoryError, StreamIdentity}; /// One aggregate event stream staged for an async transactional commit. @@ -37,6 +38,9 @@ pub struct AsyncCommitBatch<'a> { pub outbox_messages: Vec, pub read_model_plans: Vec, pub snapshots: Vec, + /// Consumer inbox receipts to record in the same transaction (the optional + /// effectively-once effect fence). Empty for the default idempotent path. + pub inbox_receipts: Vec, } impl<'a> AsyncCommitBatch<'a> { @@ -46,6 +50,7 @@ impl<'a> AsyncCommitBatch<'a> { outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots: Vec::new(), + inbox_receipts: Vec::new(), } } diff --git a/src/repository/batch.rs b/src/repository/batch.rs index eeed74369..0d38386f2 100644 --- a/src/repository/batch.rs +++ b/src/repository/batch.rs @@ -3,6 +3,7 @@ use crate::outbox::OutboxMessage; use crate::read_model::ReadModelWritePlan; use crate::snapshot::SnapshotRecord; +use super::inbox::InboxReceipt; use super::RepositoryError; /// A snapshot write staged as part of a transactional commit. @@ -17,6 +18,9 @@ pub struct CommitBatch<'a> { pub outbox_messages: Vec, pub read_model_plans: Vec, pub snapshots: Vec, + /// Consumer inbox receipts to record in the same transaction (the optional + /// effectively-once effect fence). Empty for the default idempotent path. + pub inbox_receipts: Vec, } impl<'a> CommitBatch<'a> { @@ -26,6 +30,7 @@ impl<'a> CommitBatch<'a> { outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots: Vec::new(), + inbox_receipts: Vec::new(), } } diff --git a/src/repository/inbox.rs b/src/repository/inbox.rs new file mode 100644 index 000000000..3e8027ab7 --- /dev/null +++ b/src/repository/inbox.rs @@ -0,0 +1,97 @@ +//! Consumer inbox — the optional consumer-side effect fence. +//! +//! The inbox is the consumer-side complement to the producer outbox: an optional +//! durable receipt that lets a consumer get **effectively-once local database +//! effects** on top of at-least-once transport delivery. It is *not* a read-model +//! feature (it replaces the removed `read_model_processed_messages`, which wrongly +//! coupled delivery dedupe to the projection contract — see +//! `specs/consumer-inbox-design.md`). +//! +//! An [`InboxReceipt`] is a participant in the transactional commit batch +//! (alongside aggregate events, outbox rows, read-model write plans, and +//! snapshots). The relational stores write it to an operational `consumer_inbox` +//! table in the **same transaction** as everything else in the batch — that +//! atomicity is what makes the fence real: handler effects and the receipt land +//! together or not at all. +//! +//! Default consumers stay idempotent (a replayed projection re-converges); the +//! inbox is the opt-in pattern for when exactly-once local effects are worth the +//! cost. It only fences *local transactional* effects — external side effects +//! (HTTP calls, etc.) still require handler idempotency. + +use std::time::SystemTime; + +/// A single consumer/message processing receipt. +/// +/// Identified by `(consumer, message_id)`; committed atomically with the +/// consumer's other writes so a redelivery of the same message is a no-op. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InboxReceipt { + /// The logical consumer (e.g. a projection or service name) — the dedupe scope. + pub consumer: String, + /// The transport message's stable id. + pub message_id: String, + /// When the receipt was created. + pub processed_at: SystemTime, +} + +impl InboxReceipt { + /// Create a receipt for `(consumer, message_id)`, stamped now. + pub fn new(consumer: impl Into, message_id: impl Into) -> Self { + Self { + consumer: consumer.into(), + message_id: message_id.into(), + processed_at: SystemTime::now(), + } + } + + /// The `(consumer, message_id)` dedupe key. + pub fn key(&self) -> (&str, &str) { + (&self.consumer, &self.message_id) + } +} + +/// Outcome of committing an [`InboxReceipt`]. +/// +/// A `Duplicate` is **not** an error: the message was already processed, so the +/// consumer treats it as success (acks) without re-running effects. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InboxOutcome { + /// The receipt was newly recorded (first time this message was processed). + Processed, + /// A receipt for `(consumer, message_id)` already existed; effects were skipped. + Duplicate, +} + +impl InboxOutcome { + /// Whether this is the first processing (vs a deduplicated replay). + pub fn is_processed(self) -> bool { + matches!(self, InboxOutcome::Processed) + } + + /// Whether the message was already processed (a deduplicated replay). + pub fn is_duplicate(self) -> bool { + matches!(self, InboxOutcome::Duplicate) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn receipt_key_is_consumer_and_message_id() { + let r = InboxReceipt::new("projections", "evt-1"); + assert_eq!(r.key(), ("projections", "evt-1")); + assert_eq!(r.consumer, "projections"); + assert_eq!(r.message_id, "evt-1"); + } + + #[test] + fn outcome_predicates() { + assert!(InboxOutcome::Processed.is_processed()); + assert!(!InboxOutcome::Processed.is_duplicate()); + assert!(InboxOutcome::Duplicate.is_duplicate()); + assert!(!InboxOutcome::Duplicate.is_processed()); + } +} diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 8672880cb..43122fc63 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -3,6 +3,7 @@ mod batch; mod error; mod gettable; mod identity; +mod inbox; mod repository; pub use async_repository::{ @@ -14,4 +15,5 @@ pub use batch::{CommitBatch, SnapshotWrite, TransactionalCommit}; pub use error::RepositoryError; pub use gettable::{GetMany, GetOne, Gettable}; pub use identity::StreamIdentity; +pub use inbox::{InboxOutcome, InboxReceipt}; pub use repository::{Commit, Get, Repository}; diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index ef4ba2fa4..82786a827 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -268,6 +268,7 @@ where outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots, + inbox_receipts: Vec::new(), }) .await?; @@ -307,6 +308,7 @@ where outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots, + inbox_receipts: Vec::new(), }) .await?; @@ -388,6 +390,7 @@ where outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots, + inbox_receipts: Vec::new(), })?; if let Some(version) = snapshot_version { @@ -417,6 +420,7 @@ where outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots, + inbox_receipts: Vec::new(), })?; for (aggregate, snapshot_version) in aggregates.iter_mut().zip(snapshot_versions) { @@ -528,6 +532,7 @@ where outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots, + inbox_receipts: Vec::new(), }; batch.outbox_messages.push(self.outbox); self.snap_repo.inner.repo().commit_batch(batch)?; From 266e7ee504ebbda62d4f8f65700c057a91ea4e79 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 01:59:09 -0500 Subject: [PATCH 22/39] feat(inbox): persist consumer_inbox receipts across all backends Storage model for the consumer inbox (per specs/consumer-inbox-design): - Migration: consumer_inbox operational table (PK (consumer, message_id), processed_at default) in both Postgres and SQLite. In-memory backend gains an inbox_store set. - commit_batch / commit_batch_async write batch.inbox_receipts inside the existing commit transaction (the effect fence). The (consumer, message_id) primary key is the dedupe gate: a duplicate receipt rolls the whole batch back via the new RepositoryError::DuplicateInboxReceipt, so a redelivery's effects are never double-applied (in-memory checks the staged set; SQL maps the unique violation). - New AsyncInboxStore::inbox_contains_async pre-check, implemented for in-memory / SQLite / Postgres, so a consumer can skip an already-processed message before opening a transaction. Tests prove record + pre-check + dedupe + atomic rollback (a batch with a duplicate and a fresh receipt rolls back whole) on all three backends: in-memory unit test, SQLite, and live Postgres. Implements [[model-consumer-inbox-across-persistence-implementa]] Co-Authored-By: Claude Opus 4.8 (1M context) --- migrations/postgres/0001_initial.sql | 12 +++ migrations/sqlite/0001_initial.sql | 12 +++ src/hashmap_repo/repository.rs | 85 ++++++++++++++++++- src/lib.rs | 2 +- src/postgres_repo/mod.rs | 56 +++++++++++- src/repository/async_repository.rs | 16 ++++ src/repository/error.rs | 15 ++++ src/repository/mod.rs | 2 +- src/sqlite_repo/mod.rs | 56 +++++++++++- .../read_models.rs | 1 + .../scenario.rs | 1 + tests/postgres_repository/main.rs | 50 +++++++++-- tests/sqlite_repository/main.rs | 49 +++++++++-- 13 files changed, 340 insertions(+), 17 deletions(-) diff --git a/migrations/postgres/0001_initial.sql b/migrations/postgres/0001_initial.sql index 5f72ec627..a3c643e8b 100644 --- a/migrations/postgres/0001_initial.sql +++ b/migrations/postgres/0001_initial.sql @@ -115,3 +115,15 @@ CREATE INDEX IF NOT EXISTS outbox_messages_source_idx CREATE INDEX IF NOT EXISTS outbox_messages_destination_idx ON outbox_messages (destination, status, created_at); + +-- Consumer inbox: optional effectively-once effect fence (the consumer-side +-- complement to the outbox). A receipt is written in the same transaction as the +-- consumer's other writes, so a duplicate (consumer, message_id) is a no-op replay. +CREATE TABLE IF NOT EXISTS consumer_inbox ( + consumer text NOT NULL, + message_id text NOT NULL, + processed_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (consumer, message_id), + CHECK (consumer <> ''), + CHECK (message_id <> '') +); diff --git a/migrations/sqlite/0001_initial.sql b/migrations/sqlite/0001_initial.sql index d24e68aa6..06d95060b 100644 --- a/migrations/sqlite/0001_initial.sql +++ b/migrations/sqlite/0001_initial.sql @@ -85,3 +85,15 @@ CREATE INDEX IF NOT EXISTS outbox_messages_source_idx CREATE INDEX IF NOT EXISTS outbox_messages_destination_idx ON outbox_messages (destination, status, created_at); + +-- Consumer inbox: optional effectively-once effect fence (the consumer-side +-- complement to the outbox). A receipt is written in the same transaction as the +-- consumer's other writes, so a duplicate (consumer, message_id) is a no-op replay. +CREATE TABLE IF NOT EXISTS consumer_inbox ( + consumer TEXT NOT NULL, + message_id TEXT NOT NULL, + processed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (consumer, message_id), + CHECK (consumer <> ''), + CHECK (message_id <> '') +); diff --git a/src/hashmap_repo/repository.rs b/src/hashmap_repo/repository.rs index 283598861..f9e157bf3 100644 --- a/src/hashmap_repo/repository.rs +++ b/src/hashmap_repo/repository.rs @@ -19,7 +19,7 @@ use crate::read_model::{ ReadModelWritePlanStore, RelationalReadModelQueryStore, }; use crate::repository::{ - AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, AsyncTransactionalCommit, Commit, CommitBatch, GetMany, GetOne, PreparedEventAppend, RepositoryError, SnapshotWrite, StreamIdentity, TransactionalCommit, @@ -37,6 +37,8 @@ pub struct HashMapRepository { outbox_store: Arc>>, model_store: InMemoryReadModelStore, snapshot_store: InMemorySnapshotStore, + /// Consumer inbox: the set of recorded `(consumer, message_id)` receipts. + inbox_store: Arc>>, } /// In-memory outbox table handle. @@ -59,6 +61,7 @@ impl HashMapRepository { outbox_store: Arc::new(RwLock::new(HashMap::new())), model_store: InMemoryReadModelStore::new(), snapshot_store: InMemorySnapshotStore::new(), + inbox_store: Arc::new(RwLock::new(HashSet::new())), } } @@ -83,6 +86,14 @@ impl HashMapRepository { pub fn snapshot_store(&self) -> &InMemorySnapshotStore { &self.snapshot_store } + + /// Whether a consumer inbox receipt for `(consumer, message_id)` is recorded. + pub fn inbox_contains(&self, consumer: &str, message_id: &str) -> bool { + self.inbox_store + .read() + .map(|set| set.contains(&(consumer.to_string(), message_id.to_string()))) + .unwrap_or(false) + } } impl GetOne for HashMapRepository { @@ -197,11 +208,16 @@ impl AsyncTransactionalCommit for HashMapRepository { .outbox_store .write() .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; + let mut inbox_storage = self + .inbox_store + .write() + .map_err(|_| RepositoryError::LockPoisoned("async inbox write"))?; let mut staged_events = storage.clone(); let mut staged_rows = relational_rows.clone(); let mut staged_snapshots = snapshot_storage.clone(); let mut staged_outbox = outbox_storage.clone(); + let mut staged_inbox = inbox_storage.clone(); for append in &prepared { let stored_len = @@ -242,10 +258,24 @@ impl AsyncTransactionalCommit for HashMapRepository { staged_outbox.insert(id, message); } + // Inbox receipts gate effectively-once: a receipt that already exists + // (committed or duplicated in this batch) rolls the whole batch back so + // effects are not double-applied. + for receipt in batch.inbox_receipts { + let key = (receipt.consumer.clone(), receipt.message_id.clone()); + if !staged_inbox.insert(key) { + return Err(RepositoryError::DuplicateInboxReceipt { + consumer: receipt.consumer, + message_id: receipt.message_id, + }); + } + } + *storage = staged_events; *relational_rows = staged_rows; *snapshot_storage = staged_snapshots; *outbox_storage = staged_outbox; + *inbox_storage = staged_inbox; for stream in batch.streams { stream.entity.mark_committed(); @@ -256,6 +286,16 @@ impl AsyncTransactionalCommit for HashMapRepository { } } +impl AsyncInboxStore for HashMapRepository { + fn inbox_contains_async<'a>( + &'a self, + consumer: &'a str, + message_id: &'a str, + ) -> impl Future> + Send + 'a { + async move { Ok(self.inbox_contains(consumer, message_id)) } + } +} + impl TransactionalCommit for HashMapRepository { fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { reject_duplicate_streams(&batch.entities)?; @@ -279,11 +319,16 @@ impl TransactionalCommit for HashMapRepository { .outbox_store .write() .map_err(|_| RepositoryError::LockPoisoned("outbox write"))?; + let mut inbox_storage = self + .inbox_store + .write() + .map_err(|_| RepositoryError::LockPoisoned("inbox write"))?; let mut staged_events = storage.clone(); let mut staged_rows = relational_rows.clone(); let mut staged_snapshots = snapshot_storage.clone(); let mut staged_outbox = outbox_storage.clone(); + let mut staged_inbox = inbox_storage.clone(); // Phase 1: Validate all stream versions before staging any writes. for entity in &batch.entities { @@ -327,11 +372,23 @@ impl TransactionalCommit for HashMapRepository { staged_outbox.insert(id, message); } + // Inbox receipts gate effectively-once (see the async impl). + for receipt in batch.inbox_receipts { + let key = (receipt.consumer.clone(), receipt.message_id.clone()); + if !staged_inbox.insert(key) { + return Err(RepositoryError::DuplicateInboxReceipt { + consumer: receipt.consumer, + message_id: receipt.message_id, + }); + } + } + // Phase 3: Publish staged state only after all validation and staging succeeds. *storage = staged_events; *relational_rows = staged_rows; *snapshot_storage = staged_snapshots; *outbox_storage = staged_outbox; + *inbox_storage = staged_inbox; for entity in batch.entities { entity.mark_committed(); @@ -633,4 +690,30 @@ mod tests { assert_eq!(entity1.new_events().len(), 1); assert_eq!(entity2.new_events().len(), 1); } + + #[test] + fn inbox_receipts_record_dedupe_and_roll_back_atomically() { + use crate::repository::InboxReceipt; + let repo = HashMapRepository::new(); + + let mut batch = CommitBatch::empty(); + batch.inbox_receipts.push(InboxReceipt::new("proj", "m1")); + repo.commit_batch(batch).unwrap(); + assert!(repo.inbox_contains("proj", "m1")); + assert!(!repo.inbox_contains("proj", "m2")); + + // A batch with a duplicate (m1) and a fresh receipt (m2) rolls back whole. + let mut dup = CommitBatch::empty(); + dup.inbox_receipts.push(InboxReceipt::new("proj", "m1")); + dup.inbox_receipts.push(InboxReceipt::new("proj", "m2")); + let err = repo.commit_batch(dup).unwrap_err(); + assert!( + matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if message_id == "m1"), + "got {err:?}" + ); + assert!( + !repo.inbox_contains("proj", "m2"), + "the duplicate rolled the whole batch back" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 16cb1d616..6432d3cba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,7 @@ pub type SourcedResult = std::result::Result; // Re-export repository traits at crate root for convenience pub use repository::{ - AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, AsyncTransactionalCommit, Commit, CommitBatch, Get, GetMany, GetOne, Gettable, InboxOutcome, InboxReceipt, PreparedEventAppend, Repository, RepositoryError, diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 4d26359f8..3d93baf13 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -32,9 +32,9 @@ use crate::read_model::{ RowValues, RowWriteMode, Versioned, }; use crate::repository::{ - AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncSnapshotWrite, - AsyncTransactionalCommit, PreparedEventAppend, RepositoryError, StreamIdentity, + AsyncTransactionalCommit, InboxReceipt, PreparedEventAppend, RepositoryError, StreamIdentity, }; use crate::snapshot::SnapshotRecord; use crate::sqlx_repo::{ @@ -323,6 +323,10 @@ impl AsyncTransactionalCommit for PostgresRepository { } } + for receipt in &batch.inbox_receipts { + insert_inbox_receipt_in_tx(&mut tx, receipt).await?; + } + tx.commit() .await .map_err(|err| repository_storage_error("commit transaction", err))?; @@ -336,6 +340,26 @@ impl AsyncTransactionalCommit for PostgresRepository { } } +impl AsyncInboxStore for PostgresRepository { + fn inbox_contains_async<'a>( + &'a self, + consumer: &'a str, + message_id: &'a str, + ) -> impl Future> + Send + 'a { + async move { + let row = sqlx::query( + "SELECT 1 FROM consumer_inbox WHERE consumer = $1 AND message_id = $2 LIMIT 1", + ) + .bind(consumer) + .bind(message_id) + .fetch_optional(&self.pool) + .await + .map_err(|err| repository_storage_error("query consumer inbox", err))?; + Ok(row.is_some()) + } + } +} + impl AsyncReadModelWritePlanStore for PostgresRepository { fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { sql_read_model_capabilities() @@ -869,6 +893,34 @@ fn validate_sql_write_plan(plan: &ReadModelWritePlan) -> Result<(), ReadModelErr plan.validate_for(&sql_read_model_capabilities()) } +/// Record a consumer inbox receipt in the commit transaction. The +/// `(consumer, message_id)` primary key is the dedupe gate: a unique violation +/// means the message was already processed, so the whole batch rolls back and the +/// effects are not double-applied. `processed_at` defaults server-side. +async fn insert_inbox_receipt_in_tx( + tx: &mut Transaction<'_, Postgres>, + receipt: &InboxReceipt, +) -> Result<(), RepositoryError> { + let result = sqlx::query("INSERT INTO consumer_inbox (consumer, message_id) VALUES ($1, $2)") + .bind(&receipt.consumer) + .bind(&receipt.message_id) + .execute(&mut **tx) + .await; + match result { + Ok(_) => Ok(()), + Err(err) if is_postgres_unique_violation(&err) => { + Err(RepositoryError::DuplicateInboxReceipt { + consumer: receipt.consumer.clone(), + message_id: receipt.message_id.clone(), + }) + } + Err(err) => Err(repository_storage_error( + "insert consumer inbox receipt", + err, + )), + } +} + async fn begin_read_model_tx(pool: &PgPool) -> Result, ReadModelError> { pool.begin() .await diff --git a/src/repository/async_repository.rs b/src/repository/async_repository.rs index c6c5c98be..88aa13a94 100644 --- a/src/repository/async_repository.rs +++ b/src/repository/async_repository.rs @@ -98,6 +98,22 @@ pub trait AsyncTransactionalCommit: Send + Sync { ) -> impl Future> + Send + 'a; } +/// Consumer inbox read capability: check whether a `(consumer, message_id)` +/// receipt has already been recorded. +/// +/// The pre-check lets a consumer skip re-running a handler for an already-processed +/// message (and ack the redelivery) before opening a transaction. The +/// authoritative dedupe is still the receipt's `(consumer, message_id)` primary +/// key written in [`commit_batch_async`](AsyncTransactionalCommit::commit_batch_async), +/// which fences the race where two deliveries both pass the pre-check. +pub trait AsyncInboxStore: Send + Sync { + fn inbox_contains_async<'a>( + &'a self, + consumer: &'a str, + message_id: &'a str, + ) -> impl Future> + Send + 'a; +} + /// Repository trait for types that implement async stream reads and commits. pub trait AsyncRepository: AsyncGetStream + AsyncTransactionalCommit {} diff --git a/src/repository/error.rs b/src/repository/error.rs index ce38e1c39..2ec6b4575 100644 --- a/src/repository/error.rs +++ b/src/repository/error.rs @@ -19,6 +19,13 @@ pub enum RepositoryError { DuplicateOutboxMessageInBatch { id: String, }, + /// A consumer inbox receipt `(consumer, message_id)` was already recorded. + /// The commit is rolled back so the consumer's effects are not double-applied; + /// the message has already been processed (an at-least-once replay). + DuplicateInboxReceipt { + consumer: String, + message_id: String, + }, InvalidStreamIdentity { aggregate_type: String, aggregate_id: String, @@ -58,6 +65,14 @@ impl fmt::Display for RepositoryError { RepositoryError::DuplicateOutboxMessageInBatch { id } => { write!(f, "duplicate outbox message id in commit batch: {}", id) } + RepositoryError::DuplicateInboxReceipt { + consumer, + message_id, + } => write!( + f, + "consumer inbox receipt already recorded for consumer `{}`, message `{}`", + consumer, message_id + ), RepositoryError::InvalidStreamIdentity { aggregate_type, aggregate_id, diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 43122fc63..a055b2ca9 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -7,7 +7,7 @@ mod inbox; mod repository; pub use async_repository::{ - AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, AsyncTransactionalCommit, PreparedEventAppend, }; diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index 7c09dc5e2..3a1ff2f37 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -31,9 +31,9 @@ use crate::read_model::{ RowValues, RowWriteMode, Versioned, }; use crate::repository::{ - AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncSnapshotWrite, - AsyncTransactionalCommit, PreparedEventAppend, RepositoryError, StreamIdentity, + AsyncTransactionalCommit, InboxReceipt, PreparedEventAppend, RepositoryError, StreamIdentity, }; use crate::snapshot::SnapshotRecord; use crate::sqlx_repo::{ @@ -307,6 +307,10 @@ impl AsyncTransactionalCommit for SqliteRepository { } } + for receipt in &batch.inbox_receipts { + insert_inbox_receipt_in_tx(&mut tx, receipt).await?; + } + tx.commit() .await .map_err(|err| repository_storage_error("commit transaction", err))?; @@ -320,6 +324,26 @@ impl AsyncTransactionalCommit for SqliteRepository { } } +impl AsyncInboxStore for SqliteRepository { + fn inbox_contains_async<'a>( + &'a self, + consumer: &'a str, + message_id: &'a str, + ) -> impl Future> + Send + 'a { + async move { + let row = sqlx::query( + "SELECT 1 FROM consumer_inbox WHERE consumer = ? AND message_id = ? LIMIT 1", + ) + .bind(consumer) + .bind(message_id) + .fetch_optional(&self.pool) + .await + .map_err(|err| repository_storage_error("query consumer inbox", err))?; + Ok(row.is_some()) + } + } +} + impl AsyncReadModelWritePlanStore for SqliteRepository { fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { sql_read_model_capabilities() @@ -884,6 +908,34 @@ fn empty_string_as_none(value: &str) -> Option<&str> { } } +/// Record a consumer inbox receipt in the commit transaction. The +/// `(consumer, message_id)` primary key is the dedupe gate: a unique violation +/// means the message was already processed, so the whole batch rolls back and the +/// effects are not double-applied. `processed_at` defaults to `CURRENT_TIMESTAMP`. +async fn insert_inbox_receipt_in_tx( + tx: &mut Transaction<'_, Sqlite>, + receipt: &InboxReceipt, +) -> Result<(), RepositoryError> { + let result = sqlx::query("INSERT INTO consumer_inbox (consumer, message_id) VALUES (?, ?)") + .bind(&receipt.consumer) + .bind(&receipt.message_id) + .execute(&mut **tx) + .await; + match result { + Ok(_) => Ok(()), + Err(err) if is_sqlite_unique_constraint(&err) => { + Err(RepositoryError::DuplicateInboxReceipt { + consumer: receipt.consumer.clone(), + message_id: receipt.message_id.clone(), + }) + } + Err(err) => Err(repository_storage_error( + "insert consumer inbox receipt", + err, + )), + } +} + async fn insert_outbox_message_in_tx( tx: &mut Transaction<'_, Sqlite>, message: &OutboxMessage, diff --git a/tests/persistent_repository_conformance/read_models.rs b/tests/persistent_repository_conformance/read_models.rs index e6071efff..c5cd74432 100644 --- a/tests/persistent_repository_conformance/read_models.rs +++ b/tests/persistent_repository_conformance/read_models.rs @@ -168,6 +168,7 @@ where StreamIdentity::new(Seat::aggregate_type(), &seat_id).expect("identity should be valid"); let err = repo .commit_batch_async(AsyncCommitBatch { + inbox_receipts: Vec::new(), streams: vec![AsyncStreamWrite::new(identity, stale.entity_mut())], outbox_messages: Vec::new(), read_model_plans: vec![read_models diff --git a/tests/persistent_repository_conformance/scenario.rs b/tests/persistent_repository_conformance/scenario.rs index 2105ac9ac..6085a7693 100644 --- a/tests/persistent_repository_conformance/scenario.rs +++ b/tests/persistent_repository_conformance/scenario.rs @@ -179,6 +179,7 @@ where .expect("checkout identity should be valid"); let err = repo .commit_batch_async(AsyncCommitBatch { + inbox_receipts: Vec::new(), streams: vec![ AsyncStreamWrite::new(stale_identity, stale.entity_mut()), AsyncStreamWrite::new(checkout_identity.clone(), checkout.entity_mut()), diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index dc9435883..e525308b4 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -9,11 +9,11 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, - AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, - AsyncTransactionalCommit, Entity, OutboxMessageStatus, PostgresRepository, ReadModel, - ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, - StreamIdentity, TableSchemaRegistry, + sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, + AsyncOutboxStore, AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, InboxReceipt, OutboxMessageStatus, PostgresRepository, + ReadModel, ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, + SnapshotRecord, StreamIdentity, TableSchemaRegistry, }; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -75,6 +75,45 @@ fn unique_id(prefix: &str) -> String { format!("{prefix}-{nanos}-{id}") } +fn inbox_batch(receipts: Vec) -> AsyncCommitBatch<'static> { + let mut batch = AsyncCommitBatch::new(Vec::new()); + batch.inbox_receipts = receipts; + batch +} + +#[tokio::test] +async fn consumer_inbox_records_dedupes_and_fences_atomically() { + let Some((_schema, repo)) = repository().await else { + return; + }; + let c = unique_id("consumer"); + let m1 = unique_id("m1"); + let m2 = unique_id("m2"); + + assert!(!repo.inbox_contains_async(&c, &m1).await.unwrap()); + repo.commit_batch_async(inbox_batch(vec![InboxReceipt::new(&c, &m1)])) + .await + .unwrap(); + assert!(repo.inbox_contains_async(&c, &m1).await.unwrap()); + + // Duplicate receipt (m1) alongside a fresh one (m2) rolls the whole batch back. + let err = repo + .commit_batch_async(inbox_batch(vec![ + InboxReceipt::new(&c, &m1), + InboxReceipt::new(&c, &m2), + ])) + .await + .unwrap_err(); + assert!( + matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if *message_id == m1), + "duplicate receipt surfaces a typed error, got {err:?}" + ); + assert!( + !repo.inbox_contains_async(&c, &m2).await.unwrap(), + "m2 must not be recorded — the duplicate rolled the whole batch back" + ); +} + async fn bootstrap_relational_counter_table(repo: &PostgresRepository) { let mut registry = TableSchemaRegistry::new(); registry.register::().unwrap(); @@ -210,6 +249,7 @@ async fn optimistic_conflict_rolls_back_other_stream_and_snapshot() { StreamIdentity::new(CounterProjection::aggregate_type(), &other_id).unwrap(); let err = repo .commit_batch_async(AsyncCommitBatch { + inbox_receipts: Vec::new(), streams: vec![ AsyncStreamWrite::new(stale_identity, stale.entity_mut()), AsyncStreamWrite::new(other_identity.clone(), other.entity_mut()), diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index bd45306f9..48b1672a7 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -4,11 +4,11 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, - AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, - AsyncTransactionalCommit, Entity, OutboxMessageStatus, ReadModel, ReadModelWritePlanBuilder, - RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, SqliteRepository, StreamIdentity, - TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, + sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, + AsyncOutboxStore, AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, InboxReceipt, OutboxMessageStatus, ReadModel, + ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, + SqliteRepository, StreamIdentity, TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, }; #[derive(Default)] @@ -55,6 +55,44 @@ async fn repository() -> SqliteRepository { .unwrap() } +fn inbox_batch(receipts: Vec) -> AsyncCommitBatch<'static> { + let mut batch = AsyncCommitBatch::new(Vec::new()); + batch.inbox_receipts = receipts; + batch +} + +#[tokio::test] +async fn consumer_inbox_records_dedupes_and_fences_atomically() { + let repo = repository().await; + + // First receipt commits and is then visible to the pre-check. + assert!(!repo.inbox_contains_async("proj", "m1").await.unwrap()); + repo.commit_batch_async(inbox_batch(vec![InboxReceipt::new("proj", "m1")])) + .await + .unwrap(); + assert!(repo.inbox_contains_async("proj", "m1").await.unwrap()); + // The dedupe scope is the consumer: a different consumer is independent. + assert!(!repo.inbox_contains_async("other", "m1").await.unwrap()); + + // A batch carrying a duplicate receipt (m1) plus a fresh one (m2) must roll + // back entirely — the receipt is the effect fence. + let err = repo + .commit_batch_async(inbox_batch(vec![ + InboxReceipt::new("proj", "m1"), + InboxReceipt::new("proj", "m2"), + ])) + .await + .unwrap_err(); + assert!( + matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if message_id == "m1"), + "duplicate receipt surfaces a typed error, got {err:?}" + ); + assert!( + !repo.inbox_contains_async("proj", "m2").await.unwrap(), + "m2 must not be recorded — the duplicate rolled the whole batch back" + ); +} + async fn bootstrap_relational_counter_table(repo: &SqliteRepository) { let mut registry = TableSchemaRegistry::new(); registry.register::().unwrap(); @@ -174,6 +212,7 @@ async fn optimistic_conflict_rolls_back_other_stream_and_read_model_plan() { StreamIdentity::new(CounterProjection::aggregate_type(), "should-not-commit").unwrap(); let err = repo .commit_batch_async(AsyncCommitBatch { + inbox_receipts: Vec::new(), streams: vec![ AsyncStreamWrite::new(stale_identity.clone(), stale.entity_mut()), AsyncStreamWrite::new(other_identity.clone(), other.entity_mut()), From bb2209e063a1b394f23a7c20412ec7b01fdd772f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 02:19:10 -0500 Subject: [PATCH 23/39] test+fix(inbox): empty-receipt parity + shared conformance scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the adversarial review of the consumer-inbox storage model: - Parity (was: in-memory accepted empty consumer/message_id while the SQL CHECK rejected with a generic Model error): add InboxReceipt::validate() + RepositoryError::InvalidInboxReceipt, enforced in all commit paths, so every backend rejects an empty receipt with the same typed error (the SQL CHECK is now a backstop). - Move inbox coverage into the shared persistent_repository_conformance suite (new inbox.rs), run against in-memory / SQLite / Postgres, replacing the hand-duplicated per-backend tests. The scenario now proves the fence against a REAL effect: a batch with a duplicate receipt AND an outbox write rolls the outbox write back too (effectively-once), plus multi-distinct receipts, cross-consumer independence, and empty-receipt rejection. - Clarify that InboxReceipt.processed_at is advisory (stamped server-side, not persisted from the field today). Deferred (low, noted): a barrier-based concurrent same-key commit test — the fence is the consumer_inbox PRIMARY KEY (a well-defined engine contract) and the unique-violation→rollback mapping is covered sequentially. Implements [[model-consumer-inbox-across-persistence-implementa]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/hashmap_repo/repository.rs | 10 ++ src/postgres_repo/mod.rs | 1 + src/repository/error.rs | 15 ++ src/repository/inbox.rs | 31 ++++- src/sqlite_repo/mod.rs | 1 + tests/hashmap_repository_conformance/main.rs | 15 ++ .../inbox.rs | 130 ++++++++++++++++++ .../persistent_repository_conformance/mod.rs | 1 + tests/postgres_repository/main.rs | 50 +------ tests/postgres_repository_conformance/main.rs | 20 +++ tests/sqlite_repository/main.rs | 49 +------ tests/sqlite_repository_conformance/main.rs | 15 ++ 12 files changed, 252 insertions(+), 86 deletions(-) create mode 100644 tests/persistent_repository_conformance/inbox.rs diff --git a/src/hashmap_repo/repository.rs b/src/hashmap_repo/repository.rs index f9e157bf3..f432b039b 100644 --- a/src/hashmap_repo/repository.rs +++ b/src/hashmap_repo/repository.rs @@ -262,6 +262,7 @@ impl AsyncTransactionalCommit for HashMapRepository { // (committed or duplicated in this batch) rolls the whole batch back so // effects are not double-applied. for receipt in batch.inbox_receipts { + receipt.validate()?; let key = (receipt.consumer.clone(), receipt.message_id.clone()); if !staged_inbox.insert(key) { return Err(RepositoryError::DuplicateInboxReceipt { @@ -374,6 +375,7 @@ impl TransactionalCommit for HashMapRepository { // Inbox receipts gate effectively-once (see the async impl). for receipt in batch.inbox_receipts { + receipt.validate()?; let key = (receipt.consumer.clone(), receipt.message_id.clone()); if !staged_inbox.insert(key) { return Err(RepositoryError::DuplicateInboxReceipt { @@ -715,5 +717,13 @@ mod tests { !repo.inbox_contains("proj", "m2"), "the duplicate rolled the whole batch back" ); + + // An empty receipt field is rejected (parity with the SQL CHECK). + let mut invalid = CommitBatch::empty(); + invalid.inbox_receipts.push(InboxReceipt::new("", "m3")); + assert!(matches!( + repo.commit_batch(invalid).unwrap_err(), + RepositoryError::InvalidInboxReceipt { .. } + )); } } diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 3d93baf13..04f6063b8 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -901,6 +901,7 @@ async fn insert_inbox_receipt_in_tx( tx: &mut Transaction<'_, Postgres>, receipt: &InboxReceipt, ) -> Result<(), RepositoryError> { + receipt.validate()?; let result = sqlx::query("INSERT INTO consumer_inbox (consumer, message_id) VALUES ($1, $2)") .bind(&receipt.consumer) .bind(&receipt.message_id) diff --git a/src/repository/error.rs b/src/repository/error.rs index 2ec6b4575..a9cc54930 100644 --- a/src/repository/error.rs +++ b/src/repository/error.rs @@ -26,6 +26,13 @@ pub enum RepositoryError { consumer: String, message_id: String, }, + /// A consumer inbox receipt had an empty `consumer` or `message_id`. Rejected + /// uniformly across backends before any write (the relational `CHECK` + /// constraints are a defense-in-depth backstop). + InvalidInboxReceipt { + consumer: String, + message_id: String, + }, InvalidStreamIdentity { aggregate_type: String, aggregate_id: String, @@ -73,6 +80,14 @@ impl fmt::Display for RepositoryError { "consumer inbox receipt already recorded for consumer `{}`, message `{}`", consumer, message_id ), + RepositoryError::InvalidInboxReceipt { + consumer, + message_id, + } => write!( + f, + "invalid consumer inbox receipt (consumer `{}`, message `{}`): consumer and message id must be non-empty", + consumer, message_id + ), RepositoryError::InvalidStreamIdentity { aggregate_type, aggregate_id, diff --git a/src/repository/inbox.rs b/src/repository/inbox.rs index 3e8027ab7..0b30fd17e 100644 --- a/src/repository/inbox.rs +++ b/src/repository/inbox.rs @@ -21,6 +21,8 @@ use std::time::SystemTime; +use super::RepositoryError; + /// A single consumer/message processing receipt. /// /// Identified by `(consumer, message_id)`; committed atomically with the @@ -31,7 +33,9 @@ pub struct InboxReceipt { pub consumer: String, /// The transport message's stable id. pub message_id: String, - /// When the receipt was created. + /// When the receipt was created (advisory). The relational backends stamp the + /// stored `processed_at` server-side, so this value is not currently persisted; + /// it exists for in-process use and forward compatibility with retention/prune. pub processed_at: SystemTime, } @@ -49,6 +53,18 @@ impl InboxReceipt { pub fn key(&self) -> (&str, &str) { (&self.consumer, &self.message_id) } + + /// Reject an empty `consumer` or `message_id` so every backend behaves + /// identically (the relational `CHECK` constraints are a backstop). + pub fn validate(&self) -> Result<(), RepositoryError> { + if self.consumer.is_empty() || self.message_id.is_empty() { + return Err(RepositoryError::InvalidInboxReceipt { + consumer: self.consumer.clone(), + message_id: self.message_id.clone(), + }); + } + Ok(()) + } } /// Outcome of committing an [`InboxReceipt`]. @@ -94,4 +110,17 @@ mod tests { assert!(InboxOutcome::Duplicate.is_duplicate()); assert!(!InboxOutcome::Duplicate.is_processed()); } + + #[test] + fn validate_rejects_empty_fields() { + assert!(InboxReceipt::new("c", "m").validate().is_ok()); + assert!(matches!( + InboxReceipt::new("", "m").validate(), + Err(RepositoryError::InvalidInboxReceipt { .. }) + )); + assert!(matches!( + InboxReceipt::new("c", "").validate(), + Err(RepositoryError::InvalidInboxReceipt { .. }) + )); + } } diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index 3a1ff2f37..13aab12a7 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -916,6 +916,7 @@ async fn insert_inbox_receipt_in_tx( tx: &mut Transaction<'_, Sqlite>, receipt: &InboxReceipt, ) -> Result<(), RepositoryError> { + receipt.validate()?; let result = sqlx::query("INSERT INTO consumer_inbox (consumer, message_id) VALUES (?, ?)") .bind(&receipt.consumer) .bind(&receipt.message_id) diff --git a/tests/hashmap_repository_conformance/main.rs b/tests/hashmap_repository_conformance/main.rs index 1714e1586..20584a837 100644 --- a/tests/hashmap_repository_conformance/main.rs +++ b/tests/hashmap_repository_conformance/main.rs @@ -99,3 +99,18 @@ async fn worker_claim_by_ids_claims_only_requested() { ) .await; } + +#[tokio::test] +async fn consumer_inbox_records_dedupes_and_fences_with_real_effects() { + let repo = repository(); + conformance::inbox::inbox_records_dedupes_and_fences_with_real_effects( + repo.clone(), + repo.outbox_store(), + ) + .await; +} + +#[tokio::test] +async fn consumer_inbox_rejects_empty_receipt() { + conformance::inbox::inbox_rejects_empty_receipt(repository()).await; +} diff --git a/tests/persistent_repository_conformance/inbox.rs b/tests/persistent_repository_conformance/inbox.rs new file mode 100644 index 000000000..009b7e24e --- /dev/null +++ b/tests/persistent_repository_conformance/inbox.rs @@ -0,0 +1,130 @@ +//! Consumer inbox conformance: identical observable semantics across the +//! in-memory, SQLite, and Postgres backends. + +use sourced_rust::{ + AsyncCommitBatch, AsyncInboxStore, AsyncOutboxStore, AsyncTransactionalCommit, InboxReceipt, + OutboxMessage, OutboxMessageStatus, RepositoryError, +}; + +use super::scenario::unique_id; + +fn batch_with( + outbox: Vec, + receipts: Vec, +) -> AsyncCommitBatch<'static> { + let mut batch = AsyncCommitBatch::new(Vec::new()); + batch.outbox_messages = outbox; + batch.inbox_receipts = receipts; + batch +} + +async fn outbox_present(outbox: &S, id: &str) -> bool { + for status in [ + OutboxMessageStatus::Pending, + OutboxMessageStatus::InFlight, + OutboxMessageStatus::Published, + OutboxMessageStatus::Failed, + ] { + let messages = outbox + .messages_by_status_async(status) + .await + .expect("outbox status lookup should succeed"); + if messages.iter().any(|m| m.id() == id) { + return true; + } + } + false +} + +/// Records a receipt, dedupes a replay, and — crucially — fences a **real +/// effect**: a batch carrying a duplicate receipt rolls back its outbox write too. +/// Also covers multiple distinct receipts in one batch and cross-consumer +/// independence of the same message id. +pub async fn inbox_records_dedupes_and_fences_with_real_effects(repo: R, outbox: S) +where + R: AsyncInboxStore + AsyncTransactionalCommit + Clone + Send + Sync + 'static, + S: AsyncOutboxStore + Send + Sync, +{ + let consumer = unique_id("consumer"); + let m1 = unique_id("msg"); + let effect1 = unique_id("effect"); + let effect2 = unique_id("effect"); + + // Pre-check is false until the receipt is committed. + assert!(!repo.inbox_contains_async(&consumer, &m1).await.unwrap()); + + // First delivery: the receipt commits atomically with a real outbox effect. + repo.commit_batch_async(batch_with( + vec![OutboxMessage::create(&effect1, "Effect", b"{}".to_vec()).unwrap()], + vec![InboxReceipt::new(&consumer, &m1)], + )) + .await + .expect("first delivery commits"); + assert!(repo.inbox_contains_async(&consumer, &m1).await.unwrap()); + assert!( + outbox_present(&outbox, &effect1).await, + "first effect landed" + ); + + // Replay: a batch with the duplicate receipt AND a fresh effect must roll back + // the effect too — proving the receipt fences the whole transaction. + let err = repo + .commit_batch_async(batch_with( + vec![OutboxMessage::create(&effect2, "Effect", b"{}".to_vec()).unwrap()], + vec![InboxReceipt::new(&consumer, &m1)], + )) + .await + .expect_err("replay is rejected"); + assert!( + matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if *message_id == m1), + "got {err:?}" + ); + assert!( + !outbox_present(&outbox, &effect2).await, + "the duplicate rolled the real effect back — effectively-once fence" + ); + + // Multiple distinct receipts commit together. + let a = unique_id("msg"); + let b = unique_id("msg"); + repo.commit_batch_async(batch_with( + Vec::new(), + vec![ + InboxReceipt::new(&consumer, &a), + InboxReceipt::new(&consumer, &b), + ], + )) + .await + .expect("distinct receipts commit"); + assert!(repo.inbox_contains_async(&consumer, &a).await.unwrap()); + assert!(repo.inbox_contains_async(&consumer, &b).await.unwrap()); + + // The dedupe scope is the consumer: the same message id for a different + // consumer is independent. + let other = unique_id("consumer"); + repo.commit_batch_async(batch_with(Vec::new(), vec![InboxReceipt::new(&other, &m1)])) + .await + .expect("a different consumer records the same message id independently"); + assert!(repo.inbox_contains_async(&other, &m1).await.unwrap()); +} + +/// Empty `consumer` or `message_id` is rejected with the same typed error on +/// every backend (parity — the relational `CHECK` is only a backstop). +pub async fn inbox_rejects_empty_receipt(repo: R) +where + R: AsyncInboxStore + AsyncTransactionalCommit + Clone + Send + Sync + 'static, +{ + for receipt in [ + InboxReceipt::new("", unique_id("msg")), + InboxReceipt::new(unique_id("consumer"), ""), + ] { + let err = repo + .commit_batch_async(batch_with(Vec::new(), vec![receipt])) + .await + .expect_err("an empty receipt field is rejected"); + assert!( + matches!(err, RepositoryError::InvalidInboxReceipt { .. }), + "got {err:?}" + ); + } +} diff --git a/tests/persistent_repository_conformance/mod.rs b/tests/persistent_repository_conformance/mod.rs index 45e5f7938..48849e9f0 100644 --- a/tests/persistent_repository_conformance/mod.rs +++ b/tests/persistent_repository_conformance/mod.rs @@ -1,5 +1,6 @@ pub mod checkout; pub mod checkout_saga; +pub mod inbox; pub mod outbox; pub mod read_models; pub mod scenario; diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index e525308b4..8086979e0 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -9,11 +9,11 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, - AsyncOutboxStore, AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, - AsyncTransactionalCommit, Entity, InboxReceipt, OutboxMessageStatus, PostgresRepository, - ReadModel, ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, - SnapshotRecord, StreamIdentity, TableSchemaRegistry, + sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, + AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, OutboxMessageStatus, PostgresRepository, ReadModel, + ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, + StreamIdentity, TableSchemaRegistry, }; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -75,44 +75,8 @@ fn unique_id(prefix: &str) -> String { format!("{prefix}-{nanos}-{id}") } -fn inbox_batch(receipts: Vec) -> AsyncCommitBatch<'static> { - let mut batch = AsyncCommitBatch::new(Vec::new()); - batch.inbox_receipts = receipts; - batch -} - -#[tokio::test] -async fn consumer_inbox_records_dedupes_and_fences_atomically() { - let Some((_schema, repo)) = repository().await else { - return; - }; - let c = unique_id("consumer"); - let m1 = unique_id("m1"); - let m2 = unique_id("m2"); - - assert!(!repo.inbox_contains_async(&c, &m1).await.unwrap()); - repo.commit_batch_async(inbox_batch(vec![InboxReceipt::new(&c, &m1)])) - .await - .unwrap(); - assert!(repo.inbox_contains_async(&c, &m1).await.unwrap()); - - // Duplicate receipt (m1) alongside a fresh one (m2) rolls the whole batch back. - let err = repo - .commit_batch_async(inbox_batch(vec![ - InboxReceipt::new(&c, &m1), - InboxReceipt::new(&c, &m2), - ])) - .await - .unwrap_err(); - assert!( - matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if *message_id == m1), - "duplicate receipt surfaces a typed error, got {err:?}" - ); - assert!( - !repo.inbox_contains_async(&c, &m2).await.unwrap(), - "m2 must not be recorded — the duplicate rolled the whole batch back" - ); -} +// Consumer inbox semantics are covered for all backends by the shared +// `persistent_repository_conformance::inbox` scenarios. async fn bootstrap_relational_counter_table(repo: &PostgresRepository) { let mut registry = TableSchemaRegistry::new(); diff --git a/tests/postgres_repository_conformance/main.rs b/tests/postgres_repository_conformance/main.rs index 688016a3a..c2727316e 100644 --- a/tests/postgres_repository_conformance/main.rs +++ b/tests/postgres_repository_conformance/main.rs @@ -124,3 +124,23 @@ async fn worker_claim_by_ids_claims_only_requested() { ) .await; } + +#[tokio::test] +async fn consumer_inbox_records_dedupes_and_fences_with_real_effects() { + let Some(repo) = repository().await else { + return; + }; + conformance::inbox::inbox_records_dedupes_and_fences_with_real_effects( + repo.clone(), + repo.outbox_store(), + ) + .await; +} + +#[tokio::test] +async fn consumer_inbox_rejects_empty_receipt() { + let Some(repo) = repository().await else { + return; + }; + conformance::inbox::inbox_rejects_empty_receipt(repo).await; +} diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 48b1672a7..65554643e 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -4,11 +4,11 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, - AsyncOutboxStore, AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, - AsyncTransactionalCommit, Entity, InboxReceipt, OutboxMessageStatus, ReadModel, - ReadModelWritePlanBuilder, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, - SqliteRepository, StreamIdentity, TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, + sourced, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncGetStream, AsyncOutboxStore, + AsyncReadModelWritePlanCommitExt, AsyncSnapshotStore, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, OutboxMessageStatus, ReadModel, ReadModelWritePlanBuilder, + RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, SqliteRepository, StreamIdentity, + TableSchemaRegistry, OUTBOX_MESSAGES_TABLE, }; #[derive(Default)] @@ -55,43 +55,8 @@ async fn repository() -> SqliteRepository { .unwrap() } -fn inbox_batch(receipts: Vec) -> AsyncCommitBatch<'static> { - let mut batch = AsyncCommitBatch::new(Vec::new()); - batch.inbox_receipts = receipts; - batch -} - -#[tokio::test] -async fn consumer_inbox_records_dedupes_and_fences_atomically() { - let repo = repository().await; - - // First receipt commits and is then visible to the pre-check. - assert!(!repo.inbox_contains_async("proj", "m1").await.unwrap()); - repo.commit_batch_async(inbox_batch(vec![InboxReceipt::new("proj", "m1")])) - .await - .unwrap(); - assert!(repo.inbox_contains_async("proj", "m1").await.unwrap()); - // The dedupe scope is the consumer: a different consumer is independent. - assert!(!repo.inbox_contains_async("other", "m1").await.unwrap()); - - // A batch carrying a duplicate receipt (m1) plus a fresh one (m2) must roll - // back entirely — the receipt is the effect fence. - let err = repo - .commit_batch_async(inbox_batch(vec![ - InboxReceipt::new("proj", "m1"), - InboxReceipt::new("proj", "m2"), - ])) - .await - .unwrap_err(); - assert!( - matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if message_id == "m1"), - "duplicate receipt surfaces a typed error, got {err:?}" - ); - assert!( - !repo.inbox_contains_async("proj", "m2").await.unwrap(), - "m2 must not be recorded — the duplicate rolled the whole batch back" - ); -} +// Consumer inbox semantics are covered for all backends by the shared +// `persistent_repository_conformance::inbox` scenarios. async fn bootstrap_relational_counter_table(repo: &SqliteRepository) { let mut registry = TableSchemaRegistry::new(); diff --git a/tests/sqlite_repository_conformance/main.rs b/tests/sqlite_repository_conformance/main.rs index e1b148c7a..24040ca39 100644 --- a/tests/sqlite_repository_conformance/main.rs +++ b/tests/sqlite_repository_conformance/main.rs @@ -92,3 +92,18 @@ async fn worker_claim_by_ids_claims_only_requested() { ) .await; } + +#[tokio::test] +async fn consumer_inbox_records_dedupes_and_fences_with_real_effects() { + let repo = repository().await; + conformance::inbox::inbox_records_dedupes_and_fences_with_real_effects( + repo.clone(), + repo.outbox_store(), + ) + .await; +} + +#[tokio::test] +async fn consumer_inbox_rejects_empty_receipt() { + conformance::inbox::inbox_rejects_empty_receipt(repository().await).await; +} From 94f69f75555cb65b785470d79973b97cb115418b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 02:42:50 -0500 Subject: [PATCH 24/39] =?UTF-8?q?test(matrix):=20generic=20transport=C3=97?= =?UTF-8?q?persistence=20harness=20+=20in-memory=20cell?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of the distributed read-model matrix (async bus facade only, no sync path). Ungate the generic async flow helpers so they are the primary path, and add run_checkout_over_bus: drive the seat-checkout domain flow + read-model projection + query on persistence R, route the events over transport B, and assert the projected checkout screen. Validated cell: HashMapRepository × InMemoryBus. Refs [[tasks/transport-persistence-matrix]] Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/distributed_read_model/main.rs | 217 ++++++++++++++++++++++++--- 1 file changed, 200 insertions(+), 17 deletions(-) diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 292fcae8d..0d6c926fe 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -23,18 +23,15 @@ mod query_service; mod read_models; mod seat_inventory_service; -#[cfg(any(feature = "sqlite", feature = "postgres"))] use std::sync::atomic::{AtomicU64, Ordering}; use std::thread; use std::time::{Duration, Instant}; -#[cfg(any(feature = "sqlite", feature = "postgres"))] use std::time::{SystemTime, UNIX_EPOCH}; use checkout::{ checkout_command, seat_command, AddSeat, StartCheckout, CHECKOUT_SEAT_RESERVED, SEAT_RESERVED, SEAT_RESERVED_MESSAGE, }; -#[cfg(any(feature = "sqlite", feature = "postgres"))] use checkout::{ checkout_event, json_outbox_event, seat_event, CheckoutStarted, SeatAdded, SeatReservationCompleted, SeatReserved, CHECKOUT_STARTED, RESERVING_SEAT_MESSAGE, @@ -44,7 +41,6 @@ use checkout_saga_service::CheckoutSaga; use projection_service::service as projection_service; use query_service::CheckoutQueryService; use read_models::{register_schemas, CheckoutView}; -#[cfg(any(feature = "sqlite", feature = "postgres"))] use read_models::{CheckoutStepView, SeatView}; use seat_inventory_service::Seat; use serde::Serialize; @@ -56,7 +52,6 @@ use sourced_rust::{ AggregateBuilder, HashMapRepository, InMemoryQueue, InMemoryReadModelStore, OutboxWorkerThread, Queueable, }; -#[cfg(any(feature = "sqlite", feature = "postgres"))] use sourced_rust::{ AsyncAggregateBuilder, AsyncCommitBuilderExt, AsyncGetStream, AsyncOutboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, @@ -103,17 +98,14 @@ fn wait_for_checkout_state( } } -#[cfg(any(feature = "sqlite", feature = "postgres"))] static NEXT_ASYNC_FLOW_ID: AtomicU64 = AtomicU64::new(1); -#[cfg(any(feature = "sqlite", feature = "postgres"))] struct AsyncFlowIds { checkout_id: String, seat_id: String, category: String, } -#[cfg(any(feature = "sqlite", feature = "postgres"))] fn async_unique_id(prefix: &str) -> String { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -123,7 +115,9 @@ fn async_unique_id(prefix: &str) -> String { format!("{prefix}-{nanos}-{sequence}") } -#[cfg(any(feature = "sqlite", feature = "postgres"))] +// Used by the gated sqlite/postgres flow tests; the matrix uses the per-step +// helpers directly, so this is unused in a default (no-feature) build. +#[allow(dead_code)] async fn run_async_persistent_checkout_flow( checkout_repo: R, checkout_outbox: CheckoutOutbox, @@ -221,7 +215,6 @@ async fn run_async_persistent_checkout_flow( assert_eq!(loaded_seat.checkout_id, ids.checkout_id); } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn add_seat_async(repo: &R, seat_id: &str, category: &str) -> OutboxMessage where R: AsyncTransactionalCommit + Send + Sync, @@ -243,7 +236,6 @@ where outbox } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn start_checkout_async( repo: &R, checkout_id: &str, @@ -275,7 +267,6 @@ where outbox } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn reserve_started_checkout_seat_async( repo: &R, checkout_started: &OutboxMessage, @@ -312,7 +303,6 @@ where outbox } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn record_seat_reserved_async(repo: &R, seat_reserved: &OutboxMessage) -> OutboxMessage where R: Clone + AsyncGetStream + AsyncTransactionalCommit + Send + Sync + 'static, @@ -349,7 +339,6 @@ where outbox } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn project_message_async(repo: &R, message: &OutboxMessage) where R: AsyncReadModelWritePlanStore + Send + Sync, @@ -445,7 +434,7 @@ where .expect("projection read models should commit"); } -#[cfg(any(feature = "sqlite", feature = "postgres"))] +#[allow(dead_code)] async fn assert_pending_async(store: &S, message: &OutboxMessage) where S: AsyncOutboxStore + Send + Sync, @@ -461,7 +450,6 @@ where ); } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn load_checkout_screen_async( repo: &R, checkout_id: &str, @@ -490,7 +478,6 @@ where Ok(Some(checkout)) } -#[cfg(any(feature = "sqlite", feature = "postgres"))] async fn load_seat_async(repo: &R, seat_id: &str) -> Result, ReadModelError> where R: AsyncRelationalReadModelQueryStore + Send + Sync, @@ -743,3 +730,199 @@ async fn checkout_commands_can_be_grpc_service() { .expect("gRPC write-side checkout should exist"); assert_eq!(saga.status, checkout::CHECKOUT_STARTED); } + +// =================================================================== +// Transport × persistence matrix +// +// The same seat-checkout scenario over every async bus transport and every +// persistence backend. No sync path: the domain flow, projection, and query run +// on the async repository `R`, and the events travel over the `Bus` facade `B`. +// =================================================================== + +use std::collections::HashMap as StdHashMap; +use std::sync::{Arc as StdArc, Mutex as StdMutex}; + +use sourced_rust::microsvc::transport::{Bus, BusConsumer, RunOptions}; +use sourced_rust::microsvc::{Message, MessageKind}; + +/// The four checkout events in flow (causal) order, by CloudEvent/event type. +const FLOW_EVENT_TYPES: [&str; 4] = [ + seat_event::ADDED, + checkout_event::STARTED, + seat_event::RESERVED, + checkout_event::SEAT_RESERVATION_COMPLETED, +]; + +/// Messages the transport delivered to the projection sink: (name, id, payload). +type Collected = StdArc)>>>; + +fn record_message(collected: &Collected, message: &Message) { + collected.lock().unwrap().push(( + message.name().to_string(), + message.id().unwrap_or_default().to_string(), + message.payload().to_vec(), + )); +} + +/// A subscriber service that records every checkout event it receives — the +/// transport sink the bus drains into. Subscribes to all four event names. +fn build_collector() -> (StdArc>, Collected) { + let collected: Collected = StdArc::new(StdMutex::new(Vec::new())); + let (c1, c2, c3, c4) = ( + collected.clone(), + collected.clone(), + collected.clone(), + collected.clone(), + ); + let service = Service::new(()) + .event(seat_event::ADDED) + .handle(move |ctx| { + record_message(&c1, ctx.message()); + Ok(serde_json::Value::Null) + }) + .event(checkout_event::STARTED) + .handle(move |ctx| { + record_message(&c2, ctx.message()); + Ok(serde_json::Value::Null) + }) + .event(seat_event::RESERVED) + .handle(move |ctx| { + record_message(&c3, ctx.message()); + Ok(serde_json::Value::Null) + }) + .event(checkout_event::SEAT_RESERVATION_COMPLETED) + .handle(move |ctx| { + record_message(&c4, ctx.message()); + Ok(serde_json::Value::Null) + }); + (StdArc::new(service), collected) +} + +/// Generic end-to-end matrix cell: run the seat-checkout domain flow + read-model +/// projection + query on persistence `repo`, routing the events over transport +/// `bus`. `collector`/`collected` are the bus's projection sink (the caller binds +/// the subscription first for transports that require it, e.g. RabbitMQ). +async fn run_checkout_over_bus( + bus: B, + collector: StdArc>, + collected: Collected, + repo: R, + ids: AsyncFlowIds, +) where + B: Bus + BusConsumer, + R: Clone + + AsyncGetStream + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore + + AsyncTransactionalCommit + + Send + + Sync + + 'static, +{ + // 1. Domain flow on the persistence backend → the four causal events. + let seat_added = add_seat_async(&repo, &ids.seat_id, &ids.category).await; + let checkout_started = + start_checkout_async(&repo, &ids.checkout_id, &ids.seat_id, &ids.category).await; + let seat_reserved = reserve_started_checkout_seat_async(&repo, &checkout_started).await; + let reservation_completed = record_seat_reserved_async(&repo, &seat_reserved).await; + let events = [ + seat_added, + checkout_started, + seat_reserved, + reservation_completed, + ]; + + // 2. Publish every event over the transport. + for event in &events { + let message = Message::new( + event.event_type.clone(), + MessageKind::Event, + event.payload.clone(), + ) + .with_id(event.id().to_string()); + bus.publish_message(message) + .await + .expect("event should publish over the bus"); + } + + // 3. Drain the transport into the projection sink. + bus.subscribe(collector, RunOptions::idempotent()) + .await + .expect("subscriber should drain the bus"); + + // 4. Every event must have crossed the transport; project them in causal order. + let delivered: StdHashMap)> = collected + .lock() + .unwrap() + .iter() + .map(|(name, id, payload)| (name.clone(), (id.clone(), payload.clone()))) + .collect(); + for event_type in FLOW_EVENT_TYPES { + let (id, payload) = delivered + .get(event_type) + .unwrap_or_else(|| panic!("event {event_type} should arrive over the bus")); + let message = OutboxMessage::create(id.clone(), event_type, payload.clone()) + .expect("delivered event should rebuild"); + project_message_async(&repo, &message).await; + } + + // 5. Query the projected graph and assert the user-facing checkout screen. + let checkout = load_checkout_screen_async(&repo, &ids.checkout_id) + .await + .expect("checkout read model load should succeed") + .expect("checkout should be projected"); + assert_eq!(checkout.seat_id, ids.seat_id); + assert_eq!(checkout.seat_category, ids.category); + assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); + assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); + assert_eq!( + checkout + .seat + .as_ref() + .expect("checkout should include seat") + .status, + SEAT_RESERVED + ); + let mut steps: Vec<&str> = checkout + .steps + .iter() + .map(|step| step.step.as_str()) + .collect(); + steps.sort(); + assert_eq!( + steps, + vec!["seat_reservation_completed", "seat_reserved", "started"] + ); + + let seat = load_seat_async(&repo, &ids.seat_id) + .await + .expect("seat read model load should succeed") + .expect("seat should be projected"); + assert_eq!(seat.status, SEAT_RESERVED); + assert_eq!(seat.checkout_id, ids.checkout_id); +} + +fn matrix_ids(tag: &str) -> AsyncFlowIds { + AsyncFlowIds { + checkout_id: async_unique_id(&format!("checkout-{tag}")), + seat_id: async_unique_id(&format!("seat-{tag}")), + category: "balcony".to_string(), + } +} + +/// In-memory persistence × in-memory transport — the always-on matrix cell. +#[tokio::test] +async fn matrix_in_memory_persistence_over_in_memory_bus() { + use sourced_rust::microsvc::transport::InMemoryBus; + let repo = HashMapRepository::new(); + register_schemas(repo.model_store()).expect("read-model schemas should register"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + InMemoryBus::new(), + collector, + collected, + repo, + matrix_ids("inmem-inmem"), + ) + .await; +} From bb6230f732948be8700e323f4d9c543eeb81cc3e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 02:46:49 -0500 Subject: [PATCH 25/39] =?UTF-8?q?test(matrix):=20full=20transport=C3=97per?= =?UTF-8?q?sistence=20matrix=20over=20the=20bus=20facade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The distributed read-model seat-checkout scenario now runs across every async transport × persistence backend, all green against live brokers: transports : InMemoryBus, NatsBus, RabbitBus, KafkaBus, PostgresBus, Knative persistence: HashMapRepository, SqliteRepository, PostgresRepository 12 matrix cells (broker/DB cells skip when their env var is unset): in-memory & sqlite over each of InMemory/NATS/Rabbit/Kafka/Knative, in-memory & postgres-persistence over a Postgres bus / in-memory bus. Knative is a first-class transport cell: KnativeBus POSTs CloudEvents to a local cloud_events_router serving the projection sink (the HTTP/gRPC command ingress is this same Knative surface) — no broker needed. RabbitMQ binds the subscription before publishing (topic exchange drops unrouted events); NATS ensures the stream; Postgres bus ensures its tables. Shared helpers: build_collector (the transport sink), run_checkout_over_bus (pull buses), run_checkout_over_knative (HTTP), project_and_assert_checkout. All on the async bus facade — no sync path. Refs [[tasks/transport-persistence-matrix]] Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/distributed_read_model/main.rs | 360 ++++++++++++++++++++++++++- 1 file changed, 352 insertions(+), 8 deletions(-) diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 0d6c926fe..629cb5970 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -850,24 +850,40 @@ async fn run_checkout_over_bus( .await .expect("subscriber should drain the bus"); - // 4. Every event must have crossed the transport; project them in causal order. - let delivered: StdHashMap)> = collected + // 4-5. Project the transport-delivered events in causal order, then assert. + project_and_assert_checkout(&repo, &ids, &delivered_map(&collected)).await; +} + +/// Collapse the recorded deliveries into a `name -> (id, payload)` map. +fn delivered_map(collected: &Collected) -> StdHashMap)> { + collected .lock() .unwrap() .iter() .map(|(name, id, payload)| (name.clone(), (id.clone(), payload.clone()))) - .collect(); + .collect() +} + +/// Project the events the transport delivered (in causal order) into `repo`'s +/// read models, then query the graph and assert the user-facing checkout screen. +/// Shared by every transport cell (pull buses and the Knative HTTP path). +async fn project_and_assert_checkout( + repo: &R, + ids: &AsyncFlowIds, + delivered: &StdHashMap)>, +) where + R: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore + Send + Sync, +{ for event_type in FLOW_EVENT_TYPES { let (id, payload) = delivered .get(event_type) - .unwrap_or_else(|| panic!("event {event_type} should arrive over the bus")); + .unwrap_or_else(|| panic!("event {event_type} should arrive over the transport")); let message = OutboxMessage::create(id.clone(), event_type, payload.clone()) .expect("delivered event should rebuild"); - project_message_async(&repo, &message).await; + project_message_async(repo, &message).await; } - // 5. Query the projected graph and assert the user-facing checkout screen. - let checkout = load_checkout_screen_async(&repo, &ids.checkout_id) + let checkout = load_checkout_screen_async(repo, &ids.checkout_id) .await .expect("checkout read model load should succeed") .expect("checkout should be projected"); @@ -894,7 +910,7 @@ async fn run_checkout_over_bus( vec!["seat_reservation_completed", "seat_reserved", "started"] ); - let seat = load_seat_async(&repo, &ids.seat_id) + let seat = load_seat_async(repo, &ids.seat_id) .await .expect("seat read model load should succeed") .expect("seat should be projected"); @@ -926,3 +942,331 @@ async fn matrix_in_memory_persistence_over_in_memory_bus() { ) .await; } + +// ---- Persistence fixtures (read-model schemas registered/bootstrapped) ---- + +fn inmem_matrix_repo() -> HashMapRepository { + let repo = HashMapRepository::new(); + register_schemas(repo.model_store()).expect("read-model schemas should register"); + repo +} + +#[cfg(feature = "sqlite")] +async fn sqlite_matrix_repo() -> SqliteRepository { + let repo = SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .expect("sqlite matrix repo should migrate"); + let registry = read_models::table_schema_registry().expect("schemas should build"); + repo.bootstrap_table_schema_for_dev(®istry) + .await + .expect("read-model schema should bootstrap"); + repo +} + +// ---- Knative (HTTP / CloudEvents) transport cell ---- +// +// Knative produce = POST CloudEvents to a broker-ingress; consume = the platform +// delivers them over HTTP to `cloud_events_router`. Here a local router serves the +// projection sink, so the same scenario runs over the Knative transport with no +// broker. (HTTP/gRPC command ingress is this same Knative surface.) +#[cfg(feature = "http")] +async fn run_checkout_over_knative(repo: R, ids: AsyncFlowIds) +where + R: Clone + + AsyncGetStream + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore + + AsyncTransactionalCommit + + Send + + Sync + + 'static, +{ + use sourced_rust::microsvc::transport::{cloud_events_router, KnativeBus}; + + let (collector, collected) = build_collector(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("knative ingress should bind"); + let addr = listener.local_addr().expect("ingress addr"); + let server = tokio::spawn(async move { + axum::serve(listener, cloud_events_router(collector)) + .await + .expect("knative ingress should serve"); + }); + + // events_broker "" + namespace "" => POST to the router root ("/"). + let bus = KnativeBus::new(format!("http://{addr}"), "", "matrix-source", "", ""); + + let seat_added = add_seat_async(&repo, &ids.seat_id, &ids.category).await; + let checkout_started = + start_checkout_async(&repo, &ids.checkout_id, &ids.seat_id, &ids.category).await; + let seat_reserved = reserve_started_checkout_seat_async(&repo, &checkout_started).await; + let reservation_completed = record_seat_reserved_async(&repo, &seat_reserved).await; + for event in [ + &seat_added, + &checkout_started, + &seat_reserved, + &reservation_completed, + ] { + let message = Message::new( + event.event_type.clone(), + MessageKind::Event, + event.payload.clone(), + ) + .with_id(event.id().to_string()); + bus.publish_message(message) + .await + .expect("CloudEvent should POST to the Knative ingress"); + } + + project_and_assert_checkout(&repo, &ids, &delivered_map(&collected)).await; + server.abort(); +} + +// =================== Matrix cells =================== +// +// Transport axis: InMemoryBus, NatsBus, RabbitBus, KafkaBus, PostgresBus, Knative. +// Persistence axis: HashMapRepository, SqliteRepository, PostgresRepository. +// Broker/DB cells skip when their env var is unset. + +#[cfg(feature = "sqlite")] +#[tokio::test] +async fn matrix_sqlite_persistence_over_in_memory_bus() { + use sourced_rust::microsvc::transport::InMemoryBus; + let (collector, collected) = build_collector(); + run_checkout_over_bus( + InMemoryBus::new(), + collector, + collected, + sqlite_matrix_repo().await, + matrix_ids("sqlite-inmem"), + ) + .await; +} + +#[cfg(feature = "http")] +#[tokio::test] +async fn matrix_in_memory_persistence_over_knative() { + run_checkout_over_knative(inmem_matrix_repo(), matrix_ids("inmem-knative")).await; +} + +#[cfg(all(feature = "http", feature = "sqlite"))] +#[tokio::test] +async fn matrix_sqlite_persistence_over_knative() { + run_checkout_over_knative(sqlite_matrix_repo().await, matrix_ids("sqlite-knative")).await; +} + +#[cfg(feature = "nats")] +fn nats_url() -> Option { + std::env::var("NATS_URL").ok() +} + +#[cfg(feature = "nats")] +async fn nats_matrix_bus(ns: &str) -> sourced_rust::microsvc::transport::NatsBus { + let url = nats_url().expect("NATS_URL set"); + let bus = sourced_rust::microsvc::transport::NatsBus::connect(&url, "matrix", ns) + .await + .expect("nats connect") + .with_fetch_timeout(Duration::from_millis(800)); + bus.ensure_stream().await.expect("nats stream"); + bus +} + +#[cfg(feature = "nats")] +#[tokio::test] +async fn matrix_in_memory_persistence_over_nats_bus() { + if nats_url().is_none() { + return; + } + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + nats_matrix_bus(&ns).await, + collector, + collected, + inmem_matrix_repo(), + matrix_ids("inmem-nats"), + ) + .await; +} + +#[cfg(all(feature = "nats", feature = "sqlite"))] +#[tokio::test] +async fn matrix_sqlite_persistence_over_nats_bus() { + if nats_url().is_none() { + return; + } + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + nats_matrix_bus(&ns).await, + collector, + collected, + sqlite_matrix_repo().await, + matrix_ids("sqlite-nats"), + ) + .await; +} + +#[cfg(feature = "rabbitmq")] +fn amqp_url() -> Option { + std::env::var("AMQP_URL").ok() +} + +#[cfg(feature = "rabbitmq")] +async fn rabbit_matrix_bus( + ns: &str, + collector: &StdArc>, +) -> sourced_rust::microsvc::transport::RabbitBus { + let url = amqp_url().expect("AMQP_URL set"); + let bus = sourced_rust::microsvc::transport::RabbitBus::connect(&url, "matrix", ns) + .await + .expect("rabbit connect"); + // Topic exchange drops events with no bound queue, so bind before publishing. + bus.ensure_subscription(collector.as_ref()) + .await + .expect("rabbit subscription bind"); + bus +} + +#[cfg(feature = "rabbitmq")] +#[tokio::test] +async fn matrix_in_memory_persistence_over_rabbit_bus() { + if amqp_url().is_none() { + return; + } + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + let bus = rabbit_matrix_bus(&ns, &collector).await; + run_checkout_over_bus( + bus, + collector, + collected, + inmem_matrix_repo(), + matrix_ids("inmem-rabbit"), + ) + .await; +} + +#[cfg(all(feature = "rabbitmq", feature = "sqlite"))] +#[tokio::test] +async fn matrix_sqlite_persistence_over_rabbit_bus() { + if amqp_url().is_none() { + return; + } + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + let bus = rabbit_matrix_bus(&ns, &collector).await; + run_checkout_over_bus( + bus, + collector, + collected, + sqlite_matrix_repo().await, + matrix_ids("sqlite-rabbit"), + ) + .await; +} + +#[cfg(feature = "kafka")] +fn kafka_brokers() -> Option { + std::env::var("KAFKA_BROKERS").ok() +} + +#[cfg(feature = "kafka")] +async fn kafka_matrix_bus(ns: &str) -> sourced_rust::microsvc::transport::KafkaBus { + let brokers = kafka_brokers().expect("KAFKA_BROKERS set"); + sourced_rust::microsvc::transport::KafkaBus::connect(&brokers, "matrix", ns) + .await + .expect("kafka connect") + .with_fetch_timeout(Duration::from_secs(10)) +} + +#[cfg(feature = "kafka")] +#[tokio::test] +async fn matrix_in_memory_persistence_over_kafka_bus() { + if kafka_brokers().is_none() { + return; + } + let ns = async_unique_id("ns"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + kafka_matrix_bus(&ns).await, + collector, + collected, + inmem_matrix_repo(), + matrix_ids("inmem-kafka"), + ) + .await; +} + +#[cfg(all(feature = "kafka", feature = "sqlite"))] +#[tokio::test] +async fn matrix_sqlite_persistence_over_kafka_bus() { + if kafka_brokers().is_none() { + return; + } + let ns = async_unique_id("ns"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + kafka_matrix_bus(&ns).await, + collector, + collected, + sqlite_matrix_repo().await, + matrix_ids("sqlite-kafka"), + ) + .await; +} + +#[cfg(feature = "postgres")] +#[tokio::test] +async fn matrix_in_memory_persistence_over_postgres_bus() { + use sourced_rust::microsvc::transport::PostgresBus; + let Some(schema) = postgres::PostgresTestSchema::create_from_env( + "matrix_pgbus", + "skipping Postgres-bus matrix cell", + ) + .await + else { + return; + }; + let bus_pool = schema.repository().await.pool().clone(); + let bus = PostgresBus::new(bus_pool, "matrix"); + bus.ensure_tables().await.expect("postgres bus tables"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + bus, + collector, + collected, + inmem_matrix_repo(), + matrix_ids("inmem-pgbus"), + ) + .await; +} + +#[cfg(feature = "postgres")] +#[tokio::test] +async fn matrix_postgres_persistence_over_in_memory_bus() { + use sourced_rust::microsvc::transport::InMemoryBus; + let Some(schema) = postgres::PostgresTestSchema::create_from_env( + "matrix_pg", + "skipping Postgres-persistence matrix cell", + ) + .await + else { + return; + }; + let repo = schema.repository().await; + let registry = read_models::table_schema_registry().expect("schemas should build"); + repo.bootstrap_table_schema_for_dev(®istry) + .await + .expect("read-model schema should bootstrap"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + InMemoryBus::new(), + collector, + collected, + repo, + matrix_ids("pg-inmem"), + ) + .await; +} From b35f771242566ab406b597c817dcc06bcafd6789 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 03:04:11 -0500 Subject: [PATCH 26/39] =?UTF-8?q?test(matrix):=20complete=20transport?= =?UTF-8?q?=C3=97persistence=20grid=20+=20refactor=20gold-standard=20test?= =?UTF-8?q?=20onto=20the=20async=20bus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor (not delete) the gold-standard seat_checkout_saga test onto the async InMemoryBus: same services, choreography, projection, query, and assertions — the legacy InMemoryQueue/OutboxWorkerThread/Subscribable wiring is replaced by publish_pending_outbox (claim→publish→complete bridge) + bus.subscribe hops. The projection_service/query_service modules are kept. Complete the matrix to the full 6×3 grid (18 cells), all green against live brokers: { HashMap, SQLite, Postgres } persistence × { InMemoryBus, NatsBus, RabbitBus, KafkaBus, PostgresBus, Knative } transport. Postgres-persistence fixtures + Postgres-bus pairings added; broker/DB cells skip without env. Full distributed_read_model suite: 23 passed (refactored sync test + 18 matrix cells + 2 async flow tests + HTTP/gRPC command tests). Refs [[tasks/transport-persistence-matrix]] Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/distributed_read_model/main.rs | 269 +++++++++++++++++++++------ 1 file changed, 207 insertions(+), 62 deletions(-) diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 629cb5970..2dcd2f7da 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -24,8 +24,7 @@ mod read_models; mod seat_inventory_service; use std::sync::atomic::{AtomicU64, Ordering}; -use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use checkout::{ @@ -44,14 +43,10 @@ use read_models::{register_schemas, CheckoutView}; use read_models::{CheckoutStepView, SeatView}; use seat_inventory_service::Seat; use serde::Serialize; -use sourced_rust::bus::Subscribable; -use sourced_rust::microsvc::{self, Service, Session}; +use sourced_rust::microsvc::{Service, Session}; #[cfg(feature = "sqlite")] use sourced_rust::SqliteRepository; -use sourced_rust::{ - AggregateBuilder, HashMapRepository, InMemoryQueue, InMemoryReadModelStore, OutboxWorkerThread, - Queueable, -}; +use sourced_rust::{AggregateBuilder, HashMapRepository, InMemoryReadModelStore, Queueable}; use sourced_rust::{ AsyncAggregateBuilder, AsyncCommitBuilderExt, AsyncGetStream, AsyncOutboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, @@ -73,31 +68,6 @@ where .unwrap_or_else(|err| panic!("{command} should dispatch: {err:?}")); } -fn wait_for_checkout_state( - query: &CheckoutQueryService, - checkout_id: &str, - ready: impl Fn(&CheckoutView) -> bool, -) -> CheckoutView { - let deadline = Instant::now() + Duration::from_secs(10); - - loop { - if let Some(checkout) = query - .checkout_screen(checkout_id) - .expect("query should succeed") - { - if ready(&checkout) { - return checkout; - } - } - - assert!( - Instant::now() < deadline, - "timed out waiting for checkout {checkout_id}" - ); - thread::sleep(Duration::from_millis(10)); - } -} - static NEXT_ASYNC_FLOW_ID: AtomicU64 = AtomicU64::new(1); struct AsyncFlowIds { @@ -490,29 +460,61 @@ where .map(|root| SeatView::from_row(root.data).expect("seat row should hydrate"))) } -#[test] -fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { - let queue = InMemoryQueue::new(); - let poll = Duration::from_millis(5); +/// Bridge a HashMap-backed service's pending outbox onto the async bus — the +/// new-transport equivalent of the old `OutboxWorkerThread`: claim → publish → +/// complete, so each event is forwarded exactly once. +async fn publish_pending_outbox( + outbox: &sourced_rust::HashMapOutboxStore, + bus: &sourced_rust::microsvc::transport::InMemoryBus, +) { + let claimed = outbox + .claim_async(sourced_rust::ClaimOutboxMessages::new( + "matrix-outbox-bridge", + 64, + Duration::from_secs(60), + )) + .await + .expect("outbox claim should succeed"); + for message in claimed { + let bus_message = Message::new( + message.event_type.clone(), + MessageKind::Event, + message.payload.clone(), + ) + .with_id(message.id().to_string()); + bus.publish_message(bus_message) + .await + .expect("outbox event should publish to the bus"); + let claim = sourced_rust::OutboxClaimRef::from_message(&message) + .expect("claimed message should yield a claim ref"); + outbox + .complete_async(&claim) + .await + .expect("forwarded outbox message should complete"); + } +} + +/// The original gold-standard choreography, now driven over the async +/// `InMemoryBus` instead of the legacy `InMemoryQueue` / `OutboxWorkerThread` / +/// `bus::Subscribable` wiring. Same services, same projection + query, same +/// assertions — only the transport changed. +#[tokio::test] +async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { + use sourced_rust::microsvc::transport::InMemoryBus; let checkout_store = HashMapRepository::new(); let checkout_service = checkout_saga_service::service(checkout_store.clone().queued().aggregate()); - let checkout_worker = - OutboxWorkerThread::spawn(checkout_store.outbox_store(), queue.clone(), poll); - let checkout_sub = microsvc::subscribe(checkout_service.clone(), queue.new_subscriber(), poll); - let seat_store = HashMapRepository::new(); let seat_service = seat_inventory_service::service(seat_store.clone().queued().aggregate()); - let seat_worker = OutboxWorkerThread::spawn(seat_store.outbox_store(), queue.clone(), poll); - let seat_sub = microsvc::subscribe(seat_service.clone(), queue.new_subscriber(), poll); - let read_store = InMemoryReadModelStore::new(); register_schemas(&read_store).expect("relational schemas should register"); let projection_svc = projection_service(read_store.clone()); - let projection_sub = microsvc::subscribe(projection_svc.clone(), queue.new_subscriber(), poll); let query_service = CheckoutQueryService::new(read_store.clone()); + let bus = InMemoryBus::new(); + + // Commands: add the seat, start the checkout (each writes its own outbox). dispatch( &seat_service, seat_command::ADD, @@ -521,7 +523,6 @@ fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { category: "balcony".to_string(), }, ); - dispatch( &checkout_service, checkout_command::START, @@ -532,16 +533,39 @@ fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { }, ); - let checkout = wait_for_checkout_state(&query_service, "checkout-1", |checkout| { - checkout.status == CHECKOUT_SEAT_RESERVED - && checkout - .seat - .as_ref() - .is_some_and(|seat| seat.status == SEAT_RESERVED) - }); + // Hop 1: SeatAdded + CheckoutStarted reach the bus; the projection records the + // opening state and the seat service reacts to the checkout by reserving. + publish_pending_outbox(&seat_store.outbox_store(), &bus).await; + publish_pending_outbox(&checkout_store.outbox_store(), &bus).await; + bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) + .await + .expect("projection drains the opening events"); + bus.subscribe(seat_service.clone(), RunOptions::idempotent()) + .await + .expect("seat service reacts to the started checkout"); + + // Hop 2: SeatReserved reaches the bus; the saga records it; projection updates. + publish_pending_outbox(&seat_store.outbox_store(), &bus).await; + bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) + .await + .expect("projection drains the reservation"); + bus.subscribe(checkout_service.clone(), RunOptions::idempotent()) + .await + .expect("saga records the seat reservation"); + // Hop 3: SeatReservationCompleted reaches the bus; the projection finalizes. + publish_pending_outbox(&checkout_store.outbox_store(), &bus).await; + bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) + .await + .expect("projection drains the completion"); + + let checkout = query_service + .checkout_screen("checkout-1") + .expect("checkout query should succeed") + .expect("checkout should be projected"); assert_eq!(checkout.seat_id, "A-7"); assert_eq!(checkout.seat_category, "balcony"); + assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); assert_eq!( checkout @@ -589,12 +613,6 @@ fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { .unwrap(); assert_eq!(seat.status, SEAT_RESERVED); assert_eq!(seat.checkout_id, "checkout-1"); - - let _ = checkout_sub.stop(); - let _ = seat_sub.stop(); - let _ = projection_sub.stop(); - let _ = checkout_worker.stop(); - let _ = seat_worker.stop(); } #[cfg(feature = "sqlite")] @@ -930,14 +948,12 @@ fn matrix_ids(tag: &str) -> AsyncFlowIds { #[tokio::test] async fn matrix_in_memory_persistence_over_in_memory_bus() { use sourced_rust::microsvc::transport::InMemoryBus; - let repo = HashMapRepository::new(); - register_schemas(repo.model_store()).expect("read-model schemas should register"); let (collector, collected) = build_collector(); run_checkout_over_bus( InMemoryBus::new(), collector, collected, - repo, + inmem_matrix_repo(), matrix_ids("inmem-inmem"), ) .await; @@ -1270,3 +1286,132 @@ async fn matrix_postgres_persistence_over_in_memory_bus() { ) .await; } + +// ---- Remaining matrix cells: Postgres persistence + Postgres-bus pairings ---- + +#[cfg(feature = "postgres")] +async fn postgres_matrix_repo() -> Option<( + postgres::PostgresTestSchema, + sourced_rust::PostgresRepository, +)> { + let schema = postgres::PostgresTestSchema::create_from_env( + "matrix_pg", + "skipping Postgres-persistence matrix cell", + ) + .await?; + let repo = schema.repository().await; + let registry = read_models::table_schema_registry().expect("schemas should build"); + repo.bootstrap_table_schema_for_dev(®istry) + .await + .expect("read-model schema should bootstrap"); + Some((schema, repo)) +} + +#[cfg(feature = "postgres")] +async fn postgres_matrix_bus() -> Option { + use sourced_rust::microsvc::transport::PostgresBus; + let schema = postgres::PostgresTestSchema::create_from_env( + "matrix_pgbus", + "skipping Postgres-bus matrix cell", + ) + .await?; + let bus = PostgresBus::new(schema.repository().await.pool().clone(), "matrix"); + bus.ensure_tables().await.expect("postgres bus tables"); + // The schema has no Drop, so the bus's tables outlive this fixture. + Some(bus) +} + +#[cfg(all(feature = "postgres", feature = "sqlite"))] +#[tokio::test] +async fn matrix_sqlite_persistence_over_postgres_bus() { + let Some(bus) = postgres_matrix_bus().await else { + return; + }; + let (collector, collected) = build_collector(); + run_checkout_over_bus( + bus, + collector, + collected, + sqlite_matrix_repo().await, + matrix_ids("sqlite-pgbus"), + ) + .await; +} + +#[cfg(feature = "postgres")] +#[tokio::test] +async fn matrix_postgres_persistence_over_postgres_bus() { + let (Some((_schema, repo)), Some(bus)) = + (postgres_matrix_repo().await, postgres_matrix_bus().await) + else { + return; + }; + let (collector, collected) = build_collector(); + run_checkout_over_bus(bus, collector, collected, repo, matrix_ids("pg-pgbus")).await; +} + +#[cfg(all(feature = "postgres", feature = "nats"))] +#[tokio::test] +async fn matrix_postgres_persistence_over_nats_bus() { + if nats_url().is_none() { + return; + } + let Some((_schema, repo)) = postgres_matrix_repo().await else { + return; + }; + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + nats_matrix_bus(&ns).await, + collector, + collected, + repo, + matrix_ids("pg-nats"), + ) + .await; +} + +#[cfg(all(feature = "postgres", feature = "rabbitmq"))] +#[tokio::test] +async fn matrix_postgres_persistence_over_rabbit_bus() { + if amqp_url().is_none() { + return; + } + let Some((_schema, repo)) = postgres_matrix_repo().await else { + return; + }; + let ns = async_unique_id("ns").to_lowercase(); + let (collector, collected) = build_collector(); + let bus = rabbit_matrix_bus(&ns, &collector).await; + run_checkout_over_bus(bus, collector, collected, repo, matrix_ids("pg-rabbit")).await; +} + +#[cfg(all(feature = "postgres", feature = "kafka"))] +#[tokio::test] +async fn matrix_postgres_persistence_over_kafka_bus() { + if kafka_brokers().is_none() { + return; + } + let Some((_schema, repo)) = postgres_matrix_repo().await else { + return; + }; + let ns = async_unique_id("ns"); + let (collector, collected) = build_collector(); + run_checkout_over_bus( + kafka_matrix_bus(&ns).await, + collector, + collected, + repo, + matrix_ids("pg-kafka"), + ) + .await; +} + +#[cfg(all(feature = "postgres", feature = "http"))] +#[tokio::test] +async fn matrix_postgres_persistence_over_knative() { + let Some((_schema, repo)) = postgres_matrix_repo().await else { + return; + }; + run_checkout_over_knative(repo, matrix_ids("pg-knative")).await; +} From 6eb448f77d6db97aa6397a4a1831efc01f774f32 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 10:49:07 -0500 Subject: [PATCH 27/39] test(cutover): migrate transport_subscribe onto the async InMemoryBus First step of Phase 1 (legacy sync bus removal): the pub/sub transport test now publishes events to InMemoryBus and drains them via bus.subscribe, instead of Bus::from_queue(InMemoryQueue) + microsvc::subscribe. Proves the migration pattern; the legacy bus src stays until all consumers are migrated. Refs [[tasks/async-only-consolidation]] Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/microsvc/transport_subscribe.rs | 77 +++++++++++---------------- 1 file changed, 31 insertions(+), 46 deletions(-) diff --git a/tests/microsvc/transport_subscribe.rs b/tests/microsvc/transport_subscribe.rs index 4e07b1ecf..9f01e08e5 100644 --- a/tests/microsvc/transport_subscribe.rs +++ b/tests/microsvc/transport_subscribe.rs @@ -1,14 +1,12 @@ -//! Bus transport tests — subscribe (pub/sub fan-out). +//! Bus transport tests — subscribe (pub/sub fan-out), over the async `InMemoryBus`. //! -//! Uses `Bus::from_queue` for all event interactions, proving the Bus -//! abstraction works end-to-end with `microsvc::subscribe`. +//! Events are published to the bus and drained into a subscribed `Service`, +//! proving the async bus facade dispatches pub/sub events end-to-end. use std::sync::Arc; -use std::thread; -use std::time::Duration; -use sourced_rust::bus::{Bus, Event, InMemoryQueue, Subscribable}; -use sourced_rust::microsvc::{self, Service}; +use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; +use sourced_rust::microsvc::{Message, MessageKind, Service}; use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; use crate::handlers; @@ -31,48 +29,35 @@ fn counter_service() -> Arc> { ) } -#[test] -fn dispatches_from_pubsub() { - let bus = Bus::from_queue(InMemoryQueue::new()); +#[tokio::test] +async fn dispatches_from_pubsub() { + let bus = InMemoryBus::new(); let service = counter_service(); - let subscriber = bus.subscriber().new_subscriber(); - let handle = microsvc::subscribe(service.clone(), subscriber, Duration::from_millis(10)); + for (id, name, payload) in [ + ("evt-1", handlers::counter_create::COMMAND, r#"{"id":"c1"}"#), + ( + "evt-2", + handlers::counter_increment::COMMAND, + r#"{"id":"c1","amount":10}"#, + ), + ( + "evt-3", + handlers::counter_increment::COMMAND, + r#"{"id":"c1","amount":5}"#, + ), + ] { + bus.publish_message( + Message::new(name, MessageKind::Event, payload.as_bytes().to_vec()).with_id(id), + ) + .await + .expect("event should publish"); + } - // Create - bus.publish(Event::with_string_payload( - "evt-1", - "counter.create", - r#"{"id":"c1"}"#, - )) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); - - // Increment - bus.publish(Event::with_string_payload( - "evt-2", - "counter.increment", - r#"{"id":"c1","amount":10}"#, - )) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); - - // Increment again - bus.publish(Event::with_string_payload( - "evt-3", - "counter.increment", - r#"{"id":"c1","amount":5}"#, - )) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); - - // Stop the worker before reading to avoid lock contention - let stats = handle.stop().expect("transport should stop cleanly"); - assert_eq!(stats.failed, 0); - assert_eq!(stats.handled, 3); + // Drain the published events into the subscriber (create, then the increments). + bus.subscribe(service.clone(), RunOptions::idempotent()) + .await + .expect("subscriber should drain the bus"); let counter: Counter = service.repo().get("c1").unwrap().unwrap(); assert_eq!(counter.value, 15); From 1cfaef4265b51711ffcd46dced7580aef5d6fdf7 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 11:43:24 -0500 Subject: [PATCH 28/39] feat(read-model): async ReadModelWorkspace (load_async/commit_async parity) The load -> mutate -> sync -> commit workspace ergonomic existed only over the sync store traits; the async path used the bare write-plan builder. This restores parity: the mutation/sync/diff surface is store-independent, so the same `ReadModelWorkspace` now gains `load_async`/`commit_async` over the `Async{ReadModelWritePlanStore,RelationalReadModelQueryStore}` traits, plus `AsyncReadModelLoadBuilder` and `AsyncReadModelWorkspaceExt::workspace_async()`. No struct extraction or duplicated diff logic: `load`/`commit` move to small sync- and async-bound impl blocks; everything else stays shared and unbounded. Proven with async mirrors of the include-hydration and sync-roundtrip tests on `InMemoryReadModelStore` (impls both async store traits). Sync workspace API and its tests unchanged. Part of [[tasks/async-only-consolidation]] (Phase 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 23 +-- src/read_model/mod.rs | 11 +- src/read_model/session.rs | 137 +++++++++++++++--- .../read_model_relationship_includes/main.rs | 55 ++++++- 4 files changed, 186 insertions(+), 40 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6432d3cba..5d9177d53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,17 +113,18 @@ pub use queued_repo::{ // Read models: projections and read-optimized views pub use read_model::{ - ColumnDef, ColumnType, DeleteRowMutation, ExpectedVersion, ForeignKey, InMemoryReadModelStore, - IndexDef, PatchMode, PatchRowMutation, PrimaryKey, ReadModel, ReadModelAdapterCapabilities, - ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, - ReadModelLoadRequest, ReadModelMigrationArtifact, ReadModelMutation, - ReadModelQueryCapabilities, ReadModelSchema, ReadModelSchemaAdapter, - ReadModelSchemaAdapterCapabilities, ReadModelSchemaBootstrap, ReadModelSchemaIssue, - ReadModelSchemaIssueKind, ReadModelSchemaRegistry, ReadModelSchemaVerification, - ReadModelWorkspace, ReadModelWorkspaceExt, ReadModelWritePlan, ReadModelWritePlanBuilder, - ReadModelWritePlanStore, RelationalReadModel, RelationalReadModelIncludes, - RelationalReadModelQueryStore, RelationshipDef, RelationshipKind, RowKey, RowMutation, - RowPatch, RowValue, RowValues, RowWriteMode, Versioned, DEFAULT_READ_MODEL_VERSION_COLUMN, + AsyncReadModelLoadBuilder, AsyncReadModelWorkspaceExt, ColumnDef, ColumnType, + DeleteRowMutation, ExpectedVersion, ForeignKey, InMemoryReadModelStore, IndexDef, PatchMode, + PatchRowMutation, PrimaryKey, ReadModel, ReadModelAdapterCapabilities, ReadModelCommitOutcome, + ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, + ReadModelMigrationArtifact, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, + ReadModelSchemaAdapter, ReadModelSchemaAdapterCapabilities, ReadModelSchemaBootstrap, + ReadModelSchemaIssue, ReadModelSchemaIssueKind, ReadModelSchemaRegistry, + ReadModelSchemaVerification, ReadModelWorkspace, ReadModelWorkspaceExt, ReadModelWritePlan, + ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModel, + RelationalReadModelIncludes, RelationalReadModelQueryStore, RelationshipDef, RelationshipKind, + RowKey, RowMutation, RowPatch, RowValue, RowValues, RowWriteMode, Versioned, + DEFAULT_READ_MODEL_VERSION_COLUMN, }; // Neutral table/row primitives shared by read models and operational tables. diff --git a/src/read_model/mod.rs b/src/read_model/mod.rs index c0d3351c2..a89c0cf96 100644 --- a/src/read_model/mod.rs +++ b/src/read_model/mod.rs @@ -117,9 +117,10 @@ pub use schema::{ #[cfg(any(feature = "postgres", feature = "sqlite"))] pub(crate) use session::{column_name_for, key_fingerprint, validate_key, validate_row_values}; pub use session::{ - DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, - ReadModelCommitOutcome, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, - ReadModelMutation, ReadModelQueryCapabilities, ReadModelWorkspace, ReadModelWorkspaceExt, - ReadModelWritePlan, ReadModelWritePlanBuilder, ReadModelWritePlanStore, - RelationalReadModelQueryStore, RowMutation, RowPatch, RowWriteMode, + AsyncReadModelLoadBuilder, AsyncReadModelWorkspaceExt, DeleteRowMutation, ExpectedVersion, + PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, ReadModelCommitOutcome, + ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMutation, + ReadModelQueryCapabilities, ReadModelWorkspace, ReadModelWorkspaceExt, ReadModelWritePlan, + ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModelQueryStore, RowMutation, + RowPatch, RowWriteMode, }; diff --git a/src/read_model/session.rs b/src/read_model/session.rs index 06691fca9..d4d0ed9ca 100644 --- a/src/read_model/session.rs +++ b/src/read_model/session.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; use serde::Serialize; -use crate::repository::AsyncReadModelWritePlanStore; +use crate::repository::{AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore}; use super::{ ReadModelError, ReadModelSchema, RelationalReadModel, RelationalReadModelIncludes, @@ -753,16 +753,17 @@ struct TrackedModelBaseline { const INITIAL_TRACKED_ROW_VERSION: u64 = 1; /// Store-bound read-model workspace for load, mutate, sync, commit workflows. +/// +/// The mutation/sync/diff surface is store-independent; `load`/`commit` are +/// provided by sync- and async-store impl blocks below, so the same workspace +/// drives both the sync (`commit`) and async (`commit_async`) store traits. pub struct ReadModelWorkspace<'a, S> { store: &'a S, writes: ReadModelWritePlanBuilder, baselines: Vec, } -impl<'a, S> ReadModelWorkspace<'a, S> -where - S: ReadModelWritePlanStore + RelationalReadModelQueryStore, -{ +impl<'a, S> ReadModelWorkspace<'a, S> { pub fn new(store: &'a S) -> Self { Self { store, @@ -775,18 +776,6 @@ where self.writes.is_empty() } - pub fn load(&mut self, key: RowKey) -> ReadModelLoadBuilder<'_, 'a, S, M> - where - M: RelationalReadModel + RelationalReadModelIncludes, - { - ReadModelLoadBuilder { - unit: self, - key, - includes: Vec::new(), - _marker: PhantomData, - } - } - pub fn sync(&mut self, model: M) -> Result<&mut Self, ReadModelError> where M: RelationalReadModel + RelationalReadModelIncludes, @@ -930,10 +919,6 @@ where self.writes.into_write_plan() } - pub fn commit(self) -> Result { - self.writes.commit(self.store) - } - fn track_graph( &mut self, schema: ReadModelSchema, @@ -1151,6 +1136,52 @@ where } } +impl<'a, S> ReadModelWorkspace<'a, S> +where + S: ReadModelWritePlanStore + RelationalReadModelQueryStore, +{ + /// Begin a tracked load against the synchronous store traits. + pub fn load(&mut self, key: RowKey) -> ReadModelLoadBuilder<'_, 'a, S, M> + where + M: RelationalReadModel + RelationalReadModelIncludes, + { + ReadModelLoadBuilder { + unit: self, + key, + includes: Vec::new(), + _marker: PhantomData, + } + } + + /// Commit the staged write plan through the synchronous store. + pub fn commit(self) -> Result { + self.writes.commit(self.store) + } +} + +impl<'a, S> ReadModelWorkspace<'a, S> +where + S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, +{ + /// Begin a tracked load against the asynchronous store traits. + pub fn load_async(&mut self, key: RowKey) -> AsyncReadModelLoadBuilder<'_, 'a, S, M> + where + M: RelationalReadModel + RelationalReadModelIncludes, + { + AsyncReadModelLoadBuilder { + unit: self, + key, + includes: Vec::new(), + _marker: PhantomData, + } + } + + /// Commit the staged write plan through the asynchronous store. + pub async fn commit_async(self) -> Result { + self.writes.commit_async(self.store).await + } +} + /// Builder for one explicit primary-key read-model load. pub struct ReadModelLoadBuilder<'workspace, 'store, S, M> where @@ -1212,6 +1243,70 @@ pub trait ReadModelWorkspaceExt: impl ReadModelWorkspaceExt for S where S: ReadModelWritePlanStore + RelationalReadModelQueryStore {} +/// Builder for one explicit primary-key read-model load over the async store traits. +pub struct AsyncReadModelLoadBuilder<'workspace, 'store, S, M> +where + S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, +{ + unit: &'workspace mut ReadModelWorkspace<'store, S>, + key: RowKey, + includes: Vec, + _marker: PhantomData, +} + +impl<'workspace, 'store, S, M> AsyncReadModelLoadBuilder<'workspace, 'store, S, M> +where + S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, + M: RelationalReadModel + RelationalReadModelIncludes, +{ + pub fn include(mut self, relationship: impl Into) -> Self { + self.includes.push(relationship.into()); + self + } + + pub async fn one(self) -> Result>, ReadModelError> { + let request = self + .unit + .writes + .load_with::(self.key, self.includes)?; + let graph = self.unit.store.load_graph_async(request.clone()).await?; + let Some(root) = graph.root else { + return Ok(None); + }; + + let mut model = M::from_row(root.data.clone())?; + for (include_name, include_rows) in &graph.includes { + let rows = include_rows + .rows + .iter() + .map(|row| row.data.clone()) + .collect::>(); + model.hydrate_include(include_name, rows)?; + } + + self.unit + .track_graph(request.schema, root.clone(), graph.includes)?; + Ok(Some(Versioned { + data: model, + version: root.version, + })) + } +} + +/// Extension trait that starts a tracked read-model workspace from an async store. +pub trait AsyncReadModelWorkspaceExt: + AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore + Sized +{ + fn workspace_async(&self) -> ReadModelWorkspace<'_, Self> { + ReadModelWorkspace::new(self) + } +} + +impl AsyncReadModelWorkspaceExt for S where + S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore +{ +} + fn diff_rows(before: &RowValues, after: &RowValues) -> RowPatch { let mut patch = RowPatch::new(); for (column, value) in after.iter() { diff --git a/tests/read_model_relationship_includes/main.rs b/tests/read_model_relationship_includes/main.rs index 80496fb3d..38f46e069 100644 --- a/tests/read_model_relationship_includes/main.rs +++ b/tests/read_model_relationship_includes/main.rs @@ -1,8 +1,8 @@ use serde::{Deserialize, Serialize}; use sourced_rust::{ - InMemoryReadModelStore, ReadModel, ReadModelAdapterCapabilities, ReadModelCommitOutcome, - ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, - ReadModelWorkspaceExt, ReadModelWritePlan, ReadModelWritePlanStore, + AsyncReadModelWorkspaceExt, InMemoryReadModelStore, ReadModel, ReadModelAdapterCapabilities, + ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, + ReadModelQueryCapabilities, ReadModelWorkspaceExt, ReadModelWritePlan, ReadModelWritePlanStore, RelationalReadModelQueryStore, RowKey, RowValue, }; @@ -490,3 +490,52 @@ fn belongs_to_include_rejects_composite_target_primary_key() { && message.contains("single-column primary key")) ); } + +// --- Async workspace parity ----------------------------------------------- +// +// `InMemoryReadModelStore` implements the async store traits, so the same +// workspace ergonomic is available over `workspace_async()` / +// `load_async()` / `commit_async()`. These mirror the sync `include` + +// `sync`-roundtrip tests above to prove the async path is at parity. + +#[tokio::test] +async fn async_session_hydrates_has_many_include() { + let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); + let mut read_models = store.workspace_async(); + + let loaded = read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one() + .await + .unwrap() + .unwrap(); + + assert_eq!(loaded.data.weapons[0].weapon_id, "sword"); +} + +#[tokio::test] +async fn async_sync_persists_loaded_scalar_field_without_manual_patch() { + let store = store_with_player_and_weapons([]); + let mut read_models = store.workspace_async(); + let mut loaded = read_models + .load_async::(player_key("player-1")) + .one() + .await + .unwrap() + .unwrap() + .data; + loaded.display_name = "Ada Lovelace".into(); + + read_models.sync(loaded).unwrap(); + read_models.commit_async().await.unwrap(); + + let mut check = store.workspace_async(); + let reloaded = check + .load_async::(player_key("player-1")) + .one() + .await + .unwrap() + .unwrap(); + assert_eq!(reloaded.data.display_name, "Ada Lovelace"); +} From b0b0e79c48926b916c49962df750b9b5604fdb31 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 14:50:01 -0500 Subject: [PATCH 29/39] =?UTF-8?q?feat(queued-repo):=20async=20QueuedReposi?= =?UTF-8?q?tory=20=E2=80=94=20per-aggregate=20serialization=20over=20the?= =?UTF-8?q?=20async=20surface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Async paths previously bypassed QueuedRepository entirely (AsyncCommitBuilder commits straight through commit_batch_async), so two concurrent async commits to the same aggregate could interleave. This restores the queueing ability for async: `repo.queued_async().async_aggregate::()` serializes per-aggregate get/commit exactly like the sync `.queued().aggregate::()`. Lock primitive (runtime-agnostic — no tokio dep, matching the crate's RPITIT async surface): - AsyncLock / AsyncLockManager traits + InMemoryAsyncLock / InMemoryAsyncLockManager, a hand-rolled waker-based async mutex (try_lock/unlock stay sync; only acquire awaits). QueuedRepository (struct/Clone bound moved to the impls so an async lock manager is accepted): - AsyncGetStream / AsyncTransactionalCommit with the sync locking contract: reads acquire+hold the per-stream lock, commit releases on success and holds on error, multi-locks acquired in sorted/deduped order. Keyed by StreamIdentity::storage_key consistently across get/commit/unlock. - Non-locking forwards (drop-in completeness): AsyncSnapshotStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncInboxStore. - AsyncGetWithOpts / AsyncGetAllWithOpts (no_lock opt-out) + AsyncUnlockableRepository. - Queueable::queued_async() / queued_async_with(); AsyncAggregateRepository gains get_with/peek/get_all_with/peek_all/abort/unlock mirroring the sync layer. Adversarial review (3 lenses) found two latent defects in unlock(), both fixed: waking wakers while holding the std Mutex guard could (1) poison/brick the lock if a waker panics and (2) deadlock if a waker synchronously re-polls. unlock() now drains under the guard and wakes outside it; regression tests cover both. Tests: async lock unit tests (incl. re-entrant + panicking waker regressions) and tests/queued_repo_async (mutual exclusion, per-aggregate granularity, no_lock peek, abort release). Sync QueuedRepository API and its tests unchanged. Part of [[tasks/async-only-consolidation]] (Phase 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/aggregate/async_aggregate.rs | 71 +++++++ src/lib.rs | 10 +- src/lock/async_in_memory.rs | 330 +++++++++++++++++++++++++++++++ src/lock/async_lock.rs | 26 +++ src/lock/async_lock_manager.rs | 21 ++ src/lock/mod.rs | 6 + src/queued_repo/mod.rs | 3 +- src/queued_repo/repository.rs | 327 ++++++++++++++++++++++++++++-- tests/queued_repo_async/main.rs | 141 +++++++++++++ 9 files changed, 920 insertions(+), 15 deletions(-) create mode 100644 src/lock/async_in_memory.rs create mode 100644 src/lock/async_lock.rs create mode 100644 src/lock/async_lock_manager.rs create mode 100644 tests/queued_repo_async/main.rs diff --git a/src/aggregate/async_aggregate.rs b/src/aggregate/async_aggregate.rs index 51c4ea590..75d01b8ef 100644 --- a/src/aggregate/async_aggregate.rs +++ b/src/aggregate/async_aggregate.rs @@ -1,6 +1,9 @@ use std::marker::PhantomData; use crate::entity::Entity; +use crate::queued_repo::{ + AsyncGetAllWithOpts, AsyncGetWithOpts, AsyncUnlockableRepository, ReadOpts, +}; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncStreamWrite, AsyncTransactionalCommit, RepositoryError, StreamIdentity, @@ -117,3 +120,71 @@ where .await } } + +impl AsyncAggregateRepository +where + R: AsyncGetWithOpts, + A: Aggregate + Send, +{ + /// Load an aggregate with options (e.g. `ReadOpts::no_lock()` to skip the + /// queue lock when the repository is a `queued_async()` wrapper). + pub async fn get_with(&self, id: &str, opts: ReadOpts) -> Result, RepositoryError> { + let identity = stream_identity_for::(id)?; + let Some(entity) = self.repo.get_stream_with(&identity, opts).await? else { + return Ok(None); + }; + Ok(Some(hydrate::(entity)?)) + } + + /// Non-locking read (alias for `get_with(ReadOpts::no_lock())`). + pub async fn peek(&self, id: &str) -> Result, RepositoryError> { + self.get_with(id, ReadOpts::no_lock()).await + } +} + +impl AsyncAggregateRepository +where + R: AsyncGetAllWithOpts, + A: Aggregate + Send, +{ + /// Load aggregates for the provided ids with options. + pub async fn get_all_with( + &self, + ids: &[&str], + opts: ReadOpts, + ) -> Result, RepositoryError> { + let identities = ids + .iter() + .map(|id| stream_identity_for::(id)) + .collect::, _>>()?; + let entities = self.repo.get_streams_with(&identities, opts).await?; + let mut aggregates = Vec::with_capacity(entities.len()); + for entity in entities { + aggregates.push(hydrate::(entity)?); + } + Ok(aggregates) + } + + /// Non-locking multi-read (alias for `get_all_with(ReadOpts::no_lock())`). + pub async fn peek_all(&self, ids: &[&str]) -> Result, RepositoryError> { + self.get_all_with(ids, ReadOpts::no_lock()).await + } +} + +impl AsyncAggregateRepository +where + R: AsyncUnlockableRepository, + A: Aggregate, +{ + /// Release the lock held for an aggregate after an aborted load. + pub fn abort(&self, aggregate: &A) -> Result<(), RepositoryError> { + let identity = stream_identity_for::(aggregate.entity().id())?; + self.repo.unlock(&identity) + } + + /// Release the lock held for an aggregate id. + pub fn unlock(&self, id: &str) -> Result<(), RepositoryError> { + let identity = stream_identity_for::(id)?; + self.repo.unlock(&identity) + } +} diff --git a/src/lib.rs b/src/lib.rs index 5d9177d53..c66136d2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,7 +59,10 @@ pub use postgres_repo::{PostgresOutboxStore, PostgresRepository}; pub use sqlite_repo::{SqliteOutboxStore, SqliteRepository}; // Re-export lock traits and types at crate root for convenience -pub use lock::{InMemoryLock, InMemoryLockManager, Lock, LockError, LockManager}; +pub use lock::{ + AsyncLock, AsyncLockManager, InMemoryAsyncLock, InMemoryAsyncLockFuture, + InMemoryAsyncLockManager, InMemoryLock, InMemoryLockManager, Lock, LockError, LockManager, +}; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ @@ -102,6 +105,10 @@ pub use bus::Message; pub use outbox_worker::LocalEmitterPublisher; pub use queued_repo::{ + // Async WithOpts + unlock traits (async lock manager variant) + AsyncGetAllWithOpts, + AsyncGetWithOpts, + AsyncUnlockableRepository, // WithOpts traits for opting out of locking GetAllWithOpts, GetWithOpts, @@ -109,6 +116,7 @@ pub use queued_repo::{ Queueable, QueuedRepository, ReadOpts, + UnlockableRepository, }; // Read models: projections and read-optimized views diff --git a/src/lock/async_in_memory.rs b/src/lock/async_in_memory.rs new file mode 100644 index 000000000..8379d28cf --- /dev/null +++ b/src/lock/async_in_memory.rs @@ -0,0 +1,330 @@ +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; + +use super::{AsyncLock, AsyncLockManager, LockError}; + +#[derive(Default)] +struct AsyncLockState { + locked: bool, + waiters: VecDeque, +} + +/// In-memory [`AsyncLock`] backed by a `Mutex<{ locked, waiters }>`. +/// +/// The std `Mutex` is held only for the brief state check/update — never across +/// an `.await` — so it never blocks the executor. Acquisition returns a future +/// that, while the lock is held, registers the task's waker and yields +/// `Pending`; `unlock` wakes all registered waiters so they re-contend (one +/// wins, the rest re-register). Runtime-agnostic: no dependency on any async +/// runtime, matching the rest of the crate's RPITIT async surface. +pub struct InMemoryAsyncLock { + state: Mutex, +} + +impl InMemoryAsyncLock { + pub fn new() -> Self { + InMemoryAsyncLock { + state: Mutex::new(AsyncLockState::default()), + } + } +} + +impl Default for InMemoryAsyncLock { + fn default() -> Self { + Self::new() + } +} + +/// Future returned by [`InMemoryAsyncLock::lock`]. +/// +/// Borrows the lock for its lifetime; resolves once the lock is acquired. +pub struct InMemoryAsyncLockFuture<'a> { + lock: &'a InMemoryAsyncLock, +} + +impl Future for InMemoryAsyncLockFuture<'_> { + type Output = Result<(), LockError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut state = match self.lock.state.lock() { + Ok(state) => state, + Err(err) => return Poll::Ready(Err(LockError::Poisoned(err.to_string()))), + }; + if !state.locked { + state.locked = true; + Poll::Ready(Ok(())) + } else { + // Register (or refresh) this task's waker so `unlock` can wake it. + // Dedupe by `will_wake` so repeated polls without an intervening + // unlock do not accumulate duplicate wakers. + if !state + .waiters + .iter() + .any(|waker| waker.will_wake(cx.waker())) + { + state.waiters.push_back(cx.waker().clone()); + } + Poll::Pending + } + } +} + +impl AsyncLock for InMemoryAsyncLock { + fn lock(&self) -> impl Future> + Send + '_ { + InMemoryAsyncLockFuture { lock: self } + } + + fn try_lock(&self) -> Result { + let mut state = self + .state + .lock() + .map_err(|err| LockError::Poisoned(err.to_string()))?; + if state.locked { + Ok(false) + } else { + state.locked = true; + Ok(true) + } + } + + fn unlock(&self) -> Result<(), LockError> { + // Drain waiters UNDER the guard (keeping register/drain mutually + // exclusive so no wakeup is lost), then release the guard BEFORE waking. + // `Waker::wake` runs arbitrary executor code: doing it under the std + // `Mutex` would let a panicking waker poison (permanently brick) the + // lock, and a waker that synchronously re-polls would deadlock on the + // non-reentrant guard. Waking outside the critical section avoids both. + let woken = { + let mut state = self + .state + .lock() + .map_err(|err| LockError::Poisoned(err.to_string()))?; + if state.locked { + state.locked = false; + std::mem::take(&mut state.waiters) + } else { + VecDeque::new() + } + }; + // They re-contend and one wins, the rest re-register on their next poll. + for waker in woken { + waker.wake(); + } + Ok(()) + } +} + +/// In-memory [`AsyncLockManager`] backed by a `HashMap>`. +/// +/// Lazily creates one [`InMemoryAsyncLock`] per unique key and returns the same +/// `Arc` for repeated lookups — the async counterpart to +/// [`InMemoryLockManager`](super::InMemoryLockManager). +pub struct InMemoryAsyncLockManager { + locks: Mutex>>, +} + +impl InMemoryAsyncLockManager { + pub fn new() -> Self { + InMemoryAsyncLockManager { + locks: Mutex::new(HashMap::new()), + } + } +} + +impl Default for InMemoryAsyncLockManager { + fn default() -> Self { + Self::new() + } +} + +impl AsyncLockManager for InMemoryAsyncLockManager { + type Lock = InMemoryAsyncLock; + + fn get_lock(&self, id: &str) -> Result, LockError> { + let mut locks = self + .locks + .lock() + .map_err(|_| LockError::Poisoned("async lock manager map poisoned".into()))?; + Ok(locks + .entry(id.to_string()) + .or_insert_with(|| Arc::new(InMemoryAsyncLock::new())) + .clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::mpsc; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + use std::time::Duration; + + /// A `Waker` whose `wake()` re-enters the given lock via `try_lock()`, + /// modeling an inline-polling executor. The data pointer is an + /// `Arc`. + fn reentrant_waker(lock: Arc) -> Waker { + unsafe fn clone(data: *const ()) -> RawWaker { + let arc = unsafe { Arc::from_raw(data as *const InMemoryAsyncLock) }; + let cloned = Arc::clone(&arc); + std::mem::forget(arc); + RawWaker::new(Arc::into_raw(cloned) as *const (), &REENTRANT_VTABLE) + } + unsafe fn wake(data: *const ()) { + let arc = unsafe { Arc::from_raw(data as *const InMemoryAsyncLock) }; + let _ = arc.try_lock(); // re-enter from inside wake(): must not deadlock + } + unsafe fn wake_by_ref(data: *const ()) { + let arc = unsafe { Arc::from_raw(data as *const InMemoryAsyncLock) }; + let _ = arc.try_lock(); + std::mem::forget(arc); + } + unsafe fn drop_fn(data: *const ()) { + drop(unsafe { Arc::from_raw(data as *const InMemoryAsyncLock) }); + } + static REENTRANT_VTABLE: RawWakerVTable = + RawWakerVTable::new(clone, wake, wake_by_ref, drop_fn); + let raw = RawWaker::new(Arc::into_raw(lock) as *const (), &REENTRANT_VTABLE); + unsafe { Waker::from_raw(raw) } + } + + /// A `Waker` whose `wake()` panics, modeling a misbehaving executor. + fn panicking_waker() -> Waker { + unsafe fn clone(_: *const ()) -> RawWaker { + RawWaker::new(std::ptr::null(), &PANIC_VTABLE) + } + unsafe fn wake(_: *const ()) { + panic!("waker panicked in wake()"); + } + unsafe fn wake_by_ref(_: *const ()) { + panic!("waker panicked in wake_by_ref()"); + } + unsafe fn drop_fn(_: *const ()) {} + static PANIC_VTABLE: RawWakerVTable = + RawWakerVTable::new(clone, wake, wake_by_ref, drop_fn); + unsafe { Waker::from_raw(RawWaker::new(std::ptr::null(), &PANIC_VTABLE)) } + } + + /// Park `waker` on the held `lock` by polling one acquire future to `Pending`. + fn park_waker(lock: &InMemoryAsyncLock, waker: &Waker) { + let mut cx = Context::from_waker(waker); + let mut fut = std::pin::pin!(lock.lock()); + assert!(matches!(fut.as_mut().poll(&mut cx), Poll::Pending)); + } + + #[test] + fn try_lock_reflects_state() { + let lock = InMemoryAsyncLock::new(); + assert!(lock.try_lock().unwrap()); // free → acquired + assert!(!lock.try_lock().unwrap()); // held → fails + lock.unlock().unwrap(); + assert!(lock.try_lock().unwrap()); // released → acquired again + } + + #[tokio::test] + async fn lock_resolves_immediately_when_free() { + let lock = InMemoryAsyncLock::new(); + lock.lock().await.unwrap(); + assert!(!lock.try_lock().unwrap()); // now held + lock.unlock().unwrap(); + assert!(lock.try_lock().unwrap()); + } + + #[tokio::test] + async fn second_acquire_waits_until_unlock() { + let lock = Arc::new(InMemoryAsyncLock::new()); + lock.lock().await.unwrap(); + + let order = Arc::new(AtomicUsize::new(0)); + let waiter_lock = Arc::clone(&lock); + let waiter_order = Arc::clone(&order); + let waiter = tokio::spawn(async move { + waiter_lock.lock().await.unwrap(); + // Records the order in which it acquired (must be after unlock below). + waiter_order.fetch_add(1, Ordering::SeqCst) + }); + + // Give the waiter time to park on the held lock. + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + order.load(Ordering::SeqCst), + 0, + "waiter must still be parked" + ); + + lock.unlock().unwrap(); + let acquired_at = waiter.await.unwrap(); + assert_eq!(acquired_at, 0, "waiter acquired exactly once after unlock"); + assert!(!lock.try_lock().unwrap(), "waiter holds the lock"); + } + + #[test] + fn manager_returns_same_arc_per_key() { + let manager = InMemoryAsyncLockManager::new(); + let a1 = manager.get_lock("agg-1").unwrap(); + let a2 = manager.get_lock("agg-1").unwrap(); + let b = manager.get_lock("agg-2").unwrap(); + assert!(Arc::ptr_eq(&a1, &a2)); + assert!(!Arc::ptr_eq(&a1, &b)); + } + + #[tokio::test] + async fn distinct_keys_do_not_contend() { + let manager = InMemoryAsyncLockManager::new(); + let a = manager.get_lock("agg-1").unwrap(); + let b = manager.get_lock("agg-2").unwrap(); + a.lock().await.unwrap(); + // Different key acquires without waiting on `a`. + b.lock().await.unwrap(); + a.unlock().unwrap(); + b.unlock().unwrap(); + } + + // Regression: `unlock` must wake waiters OUTSIDE the held guard, so a waker + // that synchronously re-polls the lock cannot deadlock on the non-reentrant + // std `Mutex`. Without the fix this hangs; the watchdog turns that into a + // failure instead of wedging the suite. + #[test] + fn unlock_does_not_deadlock_with_reentrant_waker() { + let lock = Arc::new(InMemoryAsyncLock::new()); + assert!(lock.try_lock().unwrap()); // hold the lock + park_waker(&lock, &reentrant_waker(Arc::clone(&lock))); + + let (tx, rx) = mpsc::channel(); + let unlock_lock = Arc::clone(&lock); + std::thread::spawn(move || { + let _ = tx.send(unlock_lock.unlock()); + }); + let result = rx + .recv_timeout(Duration::from_secs(2)) + .expect("unlock deadlocked while waking a re-entrant waker"); + result.expect("unlock should succeed"); + } + + // Regression: a panicking waker must not poison the lock's mutex, because + // `unlock` releases the guard before waking. After the panic the lock is + // still usable (and was released). + #[test] + fn unlock_does_not_poison_when_a_waker_panics() { + let lock = Arc::new(InMemoryAsyncLock::new()); + assert!(lock.try_lock().unwrap()); // hold the lock + park_waker(&lock, &panicking_waker()); + + let unlock_lock = Arc::clone(&lock); + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = unlock_lock.unlock(); + })) + .is_err(); + assert!(panicked, "the panicking waker should unwind out of unlock"); + + // Not poisoned: the guard was dropped before the panicking wake ran, and + // the lock was released, so it can be acquired again. + assert!( + lock.try_lock().unwrap(), + "lock must remain usable after a waker panic" + ); + } +} diff --git a/src/lock/async_lock.rs b/src/lock/async_lock.rs new file mode 100644 index 000000000..56b847bbd --- /dev/null +++ b/src/lock/async_lock.rs @@ -0,0 +1,26 @@ +use std::future::Future; + +use super::LockError; + +/// Async counterpart to [`Lock`](super::Lock): a single lock instance whose +/// acquisition yields to the executor instead of blocking the OS thread. +/// +/// Only `lock` is asynchronous — it must `.await` (without blocking the +/// executor) until the lock becomes free. `try_lock` and `unlock` only inspect +/// or mutate lock state and wake waiters, so they stay synchronous and +/// non-blocking, mirroring the sync [`Lock`](super::Lock) trait. +/// +/// The returned future is `Send` so an async `QueuedRepository` built on this +/// lock keeps its repository futures `Send` (required by the async repo traits). +pub trait AsyncLock: Send + Sync { + /// Acquire the lock, awaiting until it becomes available. + fn lock(&self) -> impl Future> + Send + '_; + + /// Try to acquire the lock without waiting. + /// + /// Returns `Ok(true)` if acquired, `Ok(false)` if already held. + fn try_lock(&self) -> Result; + + /// Release the lock, waking any waiters so they can re-contend. + fn unlock(&self) -> Result<(), LockError>; +} diff --git a/src/lock/async_lock_manager.rs b/src/lock/async_lock_manager.rs new file mode 100644 index 000000000..d3c17e310 --- /dev/null +++ b/src/lock/async_lock_manager.rs @@ -0,0 +1,21 @@ +use std::sync::Arc; + +use super::{AsyncLock, LockError}; + +/// Async counterpart to [`LockManager`](super::LockManager): a factory for +/// per-entity (or per-key) [`AsyncLock`]s. +/// +/// An async `QueuedRepository` uses an `AsyncLockManager` to obtain a lock for +/// each aggregate stream. The default [`InMemoryAsyncLockManager`](super::InMemoryAsyncLockManager) +/// stores locks in a `HashMap`; distributed implementations might talk to +/// Redis, Postgres advisory locks, etc. +pub trait AsyncLockManager: Send + Sync { + /// The concrete async lock type returned by this manager. + type Lock: AsyncLock; + + /// Get (or create) a lock for the given identifier. + /// + /// Repeated calls with the same `id` must return the same logical lock + /// (i.e. the same `Arc` for in-memory, or the same distributed key). + fn get_lock(&self, id: &str) -> Result, LockError>; +} diff --git a/src/lock/mod.rs b/src/lock/mod.rs index 86a3456db..389bb874d 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -24,11 +24,17 @@ //! └─────────────┘ └─────────────┘ └─────────────────────┘ //! ``` +mod async_in_memory; +mod async_lock; +mod async_lock_manager; mod error; mod in_memory; mod lock; mod lock_manager; +pub use async_in_memory::{InMemoryAsyncLock, InMemoryAsyncLockFuture, InMemoryAsyncLockManager}; +pub use async_lock::AsyncLock; +pub use async_lock_manager::AsyncLockManager; pub use error::LockError; pub use in_memory::{InMemoryLock, InMemoryLockManager}; pub use lock::Lock; diff --git a/src/queued_repo/mod.rs b/src/queued_repo/mod.rs index fe22b9eac..97c334b92 100644 --- a/src/queued_repo/mod.rs +++ b/src/queued_repo/mod.rs @@ -1,5 +1,6 @@ mod repository; pub use repository::{ - GetAllWithOpts, GetWithOpts, Queueable, QueuedRepository, ReadOpts, UnlockableRepository, + AsyncGetAllWithOpts, AsyncGetWithOpts, AsyncUnlockableRepository, GetAllWithOpts, GetWithOpts, + Queueable, QueuedRepository, ReadOpts, UnlockableRepository, }; diff --git a/src/queued_repo/repository.rs b/src/queued_repo/repository.rs index 07d1707d4..0fff2f617 100644 --- a/src/queued_repo/repository.rs +++ b/src/queued_repo/repository.rs @@ -1,9 +1,24 @@ +#![expect( + clippy::manual_async_fn, + reason = "async trait impls return impl Future + Send to preserve public Send bounds" +)] + +use std::future::Future; use std::sync::Arc; use crate::entity::{Committable, Entity}; -use crate::lock::{InMemoryLockManager, Lock, LockError, LockManager}; +use crate::lock::{ + AsyncLock, AsyncLockManager, InMemoryAsyncLockManager, InMemoryLockManager, Lock, LockError, + LockManager, +}; +use crate::read_model::{ + ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, + ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, +}; use crate::repository::{ - Commit, CommitBatch, Get, GetMany, GetOne, RepositoryError, TransactionalCommit, + AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncTransactionalCommit, Commit, + CommitBatch, Get, GetMany, GetOne, RepositoryError, StreamIdentity, TransactionalCommit, }; use crate::snapshot::{SnapshotRecord, SnapshotStore}; @@ -38,12 +53,12 @@ impl ReadOpts { /// Commit releases held locks only after the inner repository succeeds. On /// commit errors, locks remain held so callers can inspect state, retry, or /// explicitly abort. -pub struct QueuedRepository { +pub struct QueuedRepository { inner: R, lock_manager: Arc, } -impl Clone for QueuedRepository { +impl Clone for QueuedRepository { fn clone(&self) -> Self { QueuedRepository { inner: self.inner.clone(), @@ -61,15 +76,7 @@ impl QueuedRepository { } } -impl QueuedRepository { - /// Create a `QueuedRepository` with a custom lock manager. - pub fn with_lock_manager(inner: R, lock_manager: L) -> Self { - QueuedRepository { - inner, - lock_manager: Arc::new(lock_manager), - } - } - +impl QueuedRepository { /// Access the inner repository. pub fn inner(&self) -> &R { &self.inner @@ -79,6 +86,16 @@ impl QueuedRepository { pub fn lock_manager(&self) -> &L { &self.lock_manager } +} + +impl QueuedRepository { + /// Create a `QueuedRepository` with a custom lock manager. + pub fn with_lock_manager(inner: R, lock_manager: L) -> Self { + QueuedRepository { + inner, + lock_manager: Arc::new(lock_manager), + } + } pub fn lock(&self, id: impl AsRef) -> Result<(), RepositoryError> { let id = id.as_ref(); @@ -253,6 +270,278 @@ impl SnapshotStore for QueuedRepository } } +// ============================================================================ +// Async variant (async lock manager): the same serialization semantics over +// the async repository trait surface. `QueuedRepository` +// is a drop-in async repository — `.queued_async().async_aggregate::()` +// serializes per-aggregate `get`/`commit` exactly like the sync variant. +// ============================================================================ + +impl QueuedRepository { + /// Create a `QueuedRepository` with a custom async lock manager. + pub fn with_async_lock_manager(inner: R, lock_manager: L) -> Self { + QueuedRepository { + inner, + lock_manager: Arc::new(lock_manager), + } + } + + // Distinct names from the sync inherent helpers: coherence cannot prove a + // type is not both a `LockManager` and an `AsyncLockManager`, so same-named + // inherent methods across the two bounded impls would be ambiguous. + fn ensure_async_lock(&self, id: &str) -> Result, RepositoryError> { + Ok(self.lock_manager.get_lock(id)?) + } + + async fn lock_async_ids_in_order( + &self, + ids: &[&str], + ) -> Result>, RepositoryError> { + let mut unique: Vec<&str> = ids.to_vec(); + unique.sort_unstable(); + unique.dedup(); + + let mut locks = Vec::with_capacity(unique.len()); + for id in unique { + let lock = self.ensure_async_lock(id)?; + lock.lock().await?; + locks.push(lock); + } + + Ok(locks) + } +} + +impl AsyncGetStream for QueuedRepository +where + R: AsyncGetStream, + L: AsyncLockManager, +{ + fn get_stream<'a>( + &'a self, + identity: &'a StreamIdentity, + ) -> impl Future, RepositoryError>> + Send + 'a { + async move { + // Acquire and HOLD the per-stream lock across the load, like the + // synchronous `GetOne`. It is released by `commit_batch_async` on + // success, or by an explicit `unlock`/`abort`. + let lock = self.ensure_async_lock(&identity.storage_key())?; + lock.lock().await?; + self.inner.get_stream(identity).await + } + } + + fn get_streams<'a>( + &'a self, + identities: &'a [StreamIdentity], + ) -> impl Future, RepositoryError>> + Send + 'a { + async move { + let keys: Vec = identities.iter().map(StreamIdentity::storage_key).collect(); + let key_refs: Vec<&str> = keys.iter().map(String::as_str).collect(); + // Sorted-order acquire prevents deadlock; locks held after return. + let _locks = self.lock_async_ids_in_order(&key_refs).await?; + self.inner.get_streams(identities).await + } + } +} + +impl AsyncTransactionalCommit for QueuedRepository +where + R: AsyncTransactionalCommit, + L: AsyncLockManager, +{ + fn commit_batch_async<'a>( + &'a self, + batch: AsyncCommitBatch<'a>, + ) -> impl Future> + Send + 'a { + async move { + // Resolve the lock handles for the committed streams. Like the sync + // `commit_batch`, this does not acquire (a prior locking load owns + // them) and releases only after the inner commit succeeds, leaving + // them held on error so callers can retry or `abort`. + let mut locks = Vec::with_capacity(batch.streams.len()); + for stream in &batch.streams { + locks.push(self.ensure_async_lock(&stream.identity.storage_key())?); + } + + let result = self.inner.commit_batch_async(batch).await; + + if result.is_ok() { + for lock in locks { + lock.unlock()?; + } + } + + result + } + } +} + +// Non-locking forwards: read models, snapshots, and the consumer inbox are not +// gated by aggregate locks (matching the sync `SnapshotStore` delegation), so a +// queued repository stays a complete drop-in for its inner async repository. + +impl AsyncSnapshotStore for QueuedRepository +where + R: AsyncSnapshotStore, + L: AsyncLockManager, +{ + fn get_snapshot_async<'a>( + &'a self, + identity: &'a StreamIdentity, + ) -> impl Future, RepositoryError>> + Send + 'a { + self.inner.get_snapshot_async(identity) + } + + fn save_snapshot_async<'a>( + &'a self, + identity: &'a StreamIdentity, + record: SnapshotRecord, + ) -> impl Future> + Send + 'a { + self.inner.save_snapshot_async(identity, record) + } + + fn delete_snapshot_async<'a>( + &'a self, + identity: &'a StreamIdentity, + ) -> impl Future> + Send + 'a { + self.inner.delete_snapshot_async(identity) + } +} + +impl AsyncReadModelWritePlanStore for QueuedRepository +where + R: AsyncReadModelWritePlanStore, + L: AsyncLockManager, +{ + fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { + self.inner.read_model_capabilities_async() + } + + fn commit_write_plan_async( + &self, + plan: ReadModelWritePlan, + ) -> impl Future> + Send + '_ { + self.inner.commit_write_plan_async(plan) + } +} + +impl AsyncRelationalReadModelQueryStore for QueuedRepository +where + R: AsyncRelationalReadModelQueryStore, + L: AsyncLockManager, +{ + fn read_model_query_capabilities_async(&self) -> ReadModelQueryCapabilities { + self.inner.read_model_query_capabilities_async() + } + + fn load_graph_async( + &self, + request: ReadModelLoadRequest, + ) -> impl Future> + Send + '_ { + self.inner.load_graph_async(request) + } +} + +impl AsyncInboxStore for QueuedRepository +where + R: AsyncInboxStore, + L: AsyncLockManager, +{ + fn inbox_contains_async<'a>( + &'a self, + consumer: &'a str, + message_id: &'a str, + ) -> impl Future> + Send + 'a { + self.inner.inbox_contains_async(consumer, message_id) + } +} + +/// Async opt-out reads for a queued repository — the async counterpart to +/// [`GetWithOpts`]. `ReadOpts::no_lock()` reads without acquiring the lock. +pub trait AsyncGetWithOpts { + fn get_stream_with<'a>( + &'a self, + identity: &'a StreamIdentity, + opts: ReadOpts, + ) -> impl Future, RepositoryError>> + Send + 'a; +} + +/// Async opt-out multi-reads — the async counterpart to [`GetAllWithOpts`]. +pub trait AsyncGetAllWithOpts { + fn get_streams_with<'a>( + &'a self, + identities: &'a [StreamIdentity], + opts: ReadOpts, + ) -> impl Future, RepositoryError>> + Send + 'a; +} + +impl AsyncGetWithOpts for QueuedRepository +where + R: AsyncGetStream, + L: AsyncLockManager, +{ + fn get_stream_with<'a>( + &'a self, + identity: &'a StreamIdentity, + opts: ReadOpts, + ) -> impl Future, RepositoryError>> + Send + 'a { + async move { + if opts.lock { + let lock = self.ensure_async_lock(&identity.storage_key())?; + lock.lock().await?; + } + self.inner.get_stream(identity).await + } + } +} + +impl AsyncGetAllWithOpts for QueuedRepository +where + R: AsyncGetStream, + L: AsyncLockManager, +{ + fn get_streams_with<'a>( + &'a self, + identities: &'a [StreamIdentity], + opts: ReadOpts, + ) -> impl Future, RepositoryError>> + Send + 'a { + async move { + if opts.lock { + let keys: Vec = + identities.iter().map(StreamIdentity::storage_key).collect(); + let key_refs: Vec<&str> = keys.iter().map(String::as_str).collect(); + let _locks = self.lock_async_ids_in_order(&key_refs).await?; + } + self.inner.get_streams(identities).await + } + } +} + +/// Async counterpart to [`UnlockableRepository`] — releasing a held lock does +/// not await, so this stays synchronous; it exists as a separate trait because +/// coherence cannot prove a type is not both a `LockManager` and an +/// `AsyncLockManager`. +pub trait AsyncUnlockableRepository { + /// Release the lock held for a stream. + /// + /// Keyed by [`StreamIdentity`] — the same key the locking `AsyncGetStream` + /// reads acquire — so an aborted load releases exactly the lock it took. + fn unlock(&self, identity: &StreamIdentity) -> Result<(), RepositoryError>; + + /// Release a lock for an aborted load (alias for [`unlock`](Self::unlock)). + fn abort(&self, identity: &StreamIdentity) -> Result<(), RepositoryError> { + self.unlock(identity) + } +} + +impl AsyncUnlockableRepository for QueuedRepository { + fn unlock(&self, identity: &StreamIdentity) -> Result<(), RepositoryError> { + self.ensure_async_lock(&identity.storage_key())?.unlock()?; + Ok(()) + } +} + /// Builder trait for wrapping a repository with queue locking. pub trait Queueable: Sized { fn queued(self) -> QueuedRepository { @@ -262,6 +551,18 @@ pub trait Queueable: Sized { fn queued_with(self, lock_manager: L) -> QueuedRepository { QueuedRepository::with_lock_manager(self, lock_manager) } + + /// Wrap with the default async lock manager (the async counterpart to + /// [`queued`](Queueable::queued)). Pair with `.async_aggregate::()` for + /// per-aggregate serialization over the async repository surface. + fn queued_async(self) -> QueuedRepository { + QueuedRepository::with_async_lock_manager(self, InMemoryAsyncLockManager::new()) + } + + /// Wrap with a custom async lock manager. + fn queued_async_with(self, lock_manager: L) -> QueuedRepository { + QueuedRepository::with_async_lock_manager(self, lock_manager) + } } impl Queueable for T {} diff --git a/tests/queued_repo_async/main.rs b/tests/queued_repo_async/main.rs new file mode 100644 index 000000000..d7a8998c1 --- /dev/null +++ b/tests/queued_repo_async/main.rs @@ -0,0 +1,141 @@ +//! Async `QueuedRepository` — per-aggregate serialization over the async +//! repository surface. Proves `.queued_async().async_aggregate::()` engages +//! the async lock on `get`/`commit` exactly like the sync `.queued()` path, +//! plus per-aggregate granularity, the `no_lock` opt-out, and explicit abort. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use sourced_rust::{ + sourced, AsyncAggregateBuilder, AsyncAggregateRepository, Entity, HashMapRepository, + InMemoryAsyncLockManager, Queueable, +}; + +#[derive(Default)] +struct Counter { + entity: Entity, + value: i32, +} + +#[sourced(entity, aggregate_type = "queued.counter")] +impl Counter { + #[event("Created")] + fn create(&mut self, id: String) { + self.entity.set_id(&id); + } + + #[event("Incremented")] + fn increment(&mut self, id: String, by: i32) { + self.entity.set_id(&id); + self.value += by; + } +} + +type QueuedCounterRepo = AsyncAggregateRepository< + sourced_rust::QueuedRepository, + Counter, +>; + +fn queued_repo() -> Arc { + Arc::new( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ) +} + +async fn seed(repo: &QueuedCounterRepo, id: &str) { + let mut counter = Counter::default(); + counter.create(id.into()).unwrap(); + // A bare commit (no prior locking load) takes the lock handle and releases + // it on success — it does not deadlock and leaves the lock free. + repo.commit(&mut counter).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn second_load_waits_until_first_commits() { + let repo = queued_repo(); + seed(&repo, "c1").await; + + // Task 1 loads (acquires + HOLDS the per-stream lock). + let mut held = repo.get("c1").await.unwrap().unwrap(); + held.increment("c1".into(), 5).unwrap(); + + // Task 2 tries to load the same aggregate; it must park on the held lock. + let acquired = Arc::new(AtomicBool::new(false)); + let task_repo = Arc::clone(&repo); + let task_flag = Arc::clone(&acquired); + let task2 = tokio::spawn(async move { + let loaded = task_repo.get("c1").await.unwrap().unwrap(); + task_flag.store(true, Ordering::SeqCst); + loaded + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !acquired.load(Ordering::SeqCst), + "the second load must block while the first holds the lock" + ); + + // Committing the first load releases the lock, unblocking task 2. + repo.commit(&mut held).await.unwrap(); + let loaded = task2.await.unwrap(); + + assert!(acquired.load(Ordering::SeqCst)); + // Task 2 loaded AFTER the commit, so it observes the first increment. + assert_eq!( + loaded.value, 5, + "serialized load sees the prior writer's committed state" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn distinct_aggregates_do_not_block_each_other() { + let repo = queued_repo(); + seed(&repo, "a").await; + seed(&repo, "b").await; + + // Hold the lock on "a". + let _held_a = repo.get("a").await.unwrap().unwrap(); + + // Loading a different aggregate must not wait on "a"'s lock. + let got_b = tokio::time::timeout(Duration::from_millis(500), repo.get("b")) + .await + .expect("loading a distinct aggregate must not block") + .unwrap(); + assert!(got_b.is_some()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn peek_reads_without_acquiring_the_lock() { + let repo = queued_repo(); + seed(&repo, "c1").await; + + // Hold the lock on "c1". + let _held = repo.get("c1").await.unwrap().unwrap(); + + // A no-lock peek of the same aggregate must not block. + let peeked = tokio::time::timeout(Duration::from_millis(500), repo.peek("c1")) + .await + .expect("no_lock peek must not block on a held lock") + .unwrap(); + assert!(peeked.is_some()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn abort_releases_a_held_lock() { + let repo = queued_repo(); + seed(&repo, "c1").await; + + // Load (acquire) then abort without committing. + let held = repo.get("c1").await.unwrap().unwrap(); + repo.abort(&held).unwrap(); + + // The lock is free again: a subsequent load must not block. + let reloaded = tokio::time::timeout(Duration::from_millis(500), repo.get("c1")) + .await + .expect("load after abort must not block") + .unwrap(); + assert!(reloaded.is_some()); +} From f9d175fd45d7f5a99e7269885f52bb2a96eda5da Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 16:57:53 -0500 Subject: [PATCH 30/39] test(cutover): migrate transport_listen onto the async InMemoryBus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the legacy `Bus::from_queue`/`microsvc::listen` queue tests with the async `InMemoryBus` + `BusConsumer::listen` (competing-consumer queues keyed by command name). The legacy `stats.handled`/`stats.failed` handle has no async analogue, so: - success is asserted via domain outcomes (committed aggregate state), not counts; - failure tolerance is asserted by showing the consumer drains past a failing message and still processes the rest; - metadata->Session is verified through `whoami` over the bus (works via run_source -> dispatch_message -> message_to_session), with a negative control under FailurePolicy::Stop; - arbitrary queue names ("counters"/"creates") become command-name routing, so two services on one bus consume disjoint command queues without competing. Confirms Phase 1 needs no new runtime capability — metadata->Session already works and the stats gap is a test-rewrite. microsvc crate: 15 passed. Part of [[tasks/async-only-consolidation]] (Phase 1). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/microsvc/transport_listen.rs | 264 ++++++++++++++--------------- 1 file changed, 129 insertions(+), 135 deletions(-) diff --git a/tests/microsvc/transport_listen.rs b/tests/microsvc/transport_listen.rs index 52d0971b4..bb6dfdefd 100644 --- a/tests/microsvc/transport_listen.rs +++ b/tests/microsvc/transport_listen.rs @@ -1,15 +1,17 @@ -//! Bus transport tests — listen (point-to-point queue consumption). +//! Bus transport tests — listen (point-to-point queue consumption), over the +//! async `InMemoryBus`. //! -//! Uses `Bus::from_queue` for all queue interactions, proving the Bus -//! abstraction works end-to-end with `microsvc::listen`. +//! Commands are sent to the bus and drained into a `Service` via `listen` +//! (competing-consumer queues keyed by command name). These mirror the former +//! legacy-bus `microsvc::listen` tests: the old `stats.handled`/`stats.failed` +//! handle has no async analogue, so success is asserted through domain outcomes +//! (committed aggregate state) and failures through the run's `FailurePolicy`. use std::sync::Arc; -use std::thread; -use std::time::Duration; use serde_json::json; -use sourced_rust::bus::{Bus, Event, InMemoryQueue}; -use sourced_rust::microsvc::{self, Service, Session}; +use sourced_rust::microsvc::transport::{Bus, BusConsumer, FailurePolicy, InMemoryBus, RunOptions}; +use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; use crate::handlers; @@ -25,178 +27,170 @@ fn counter_service() -> Arc> { )) } -#[test] -fn dispatches_from_queue() { - let bus = Bus::from_queue(InMemoryQueue::new()); - let service = counter_service(); - - let handle = microsvc::listen( - service.clone(), - "counters", - bus.subscriber().clone(), - Duration::from_millis(10), - ); - - bus.send( - "counters", - Event::with_string_payload("cmd-1", "counter.create", r#"{"id":"c1"}"#), - ) - .unwrap(); +fn command(name: &str, id: &str, payload: &str) -> Message { + Message::new(name, MessageKind::Command, payload.as_bytes().to_vec()).with_id(id) +} - thread::sleep(Duration::from_millis(200)); +#[tokio::test] +async fn dispatches_from_queue() { + let bus = InMemoryBus::new(); + let service = counter_service(); - bus.send( - "counters", - Event::with_string_payload("cmd-2", "counter.increment", r#"{"id":"c1","amount":10}"#), - ) - .unwrap(); + bus.send_message(command("counter.create", "cmd-1", r#"{"id":"c1"}"#)) + .await + .expect("create should enqueue"); + bus.send_message(command( + "counter.increment", + "cmd-2", + r#"{"id":"c1","amount":10}"#, + )) + .await + .expect("increment should enqueue"); - thread::sleep(Duration::from_millis(200)); + // Per-command queues drain in registration order (create before increment). + bus.listen(service.clone(), RunOptions::idempotent()) + .await + .expect("listen should drain the command queues"); let counter: Counter = service.repo().get("c1").unwrap().unwrap(); assert_eq!(counter.value, 10); - - let stats = handle.stop().expect("transport should stop cleanly"); - assert_eq!(stats.handled, 2); - assert_eq!(stats.failed, 0); } -#[test] -fn tracks_failures() { - let bus = Bus::from_queue(InMemoryQueue::new()); +#[tokio::test] +async fn tolerates_handler_failures_and_keeps_processing() { + let bus = InMemoryBus::new(); let service = counter_service(); - let handle = microsvc::listen( - service.clone(), - "counters", - bus.subscriber().clone(), - Duration::from_millis(10), - ); - - bus.send( - "counters", - Event::with_string_payload( - "cmd-1", - "counter.increment", - r#"{"id":"nonexistent","amount":1}"#, - ), - ) - .unwrap(); + // A failing message (increment a counter that was never created → NotFound) + // must not wedge the consumer: it should drain and still process the rest. + // NotFound is retryable → nacked (a no-op for the in-memory bus), so the run + // completes. (We deliberately do not re-read the failed id: the sync queued + // `get` inside the handler holds that aggregate's lock once the handler + // errors before committing, so re-reading it would block.) + bus.send_message(command( + "counter.increment", + "bad", + r#"{"id":"nonexistent","amount":1}"#, + )) + .await + .expect("bad increment should enqueue"); + bus.send_message(command("counter.create", "good-create", r#"{"id":"c2"}"#)) + .await + .expect("create should enqueue"); + bus.send_message(command( + "counter.increment", + "good-inc", + r#"{"id":"c2","amount":7}"#, + )) + .await + .expect("good increment should enqueue"); - thread::sleep(Duration::from_millis(200)); + bus.listen(service.clone(), RunOptions::idempotent()) + .await + .expect("listen should drain past the failed message"); - let stats = handle.stop().expect("transport should stop cleanly"); - assert_eq!(stats.handled, 0); - assert_eq!(stats.failed, 1); + // The good aggregate was still created and incremented, proving the failure + // did not stop the consumer. + let c2: Counter = service.repo().get("c2").unwrap().unwrap(); + assert_eq!(c2.value, 7); } -#[test] -fn coexists_with_direct_dispatch() { - let bus = Bus::from_queue(InMemoryQueue::new()); +#[tokio::test] +async fn coexists_with_direct_dispatch() { + let bus = InMemoryBus::new(); let service = counter_service(); - let handle = microsvc::listen( - service.clone(), - "counters", - bus.subscriber().clone(), - Duration::from_millis(10), - ); - - // Create c1 via bus - bus.send( - "counters", - Event::with_string_payload("cmd-1", "counter.create", r#"{"id":"c1"}"#), - ) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); + // c1 created via the bus. + bus.send_message(command("counter.create", "cmd-1", r#"{"id":"c1"}"#)) + .await + .expect("create should enqueue"); + bus.listen(service.clone(), RunOptions::idempotent()) + .await + .expect("listen should drain c1's create"); - // Create c2 via direct dispatch + // c2 created via direct dispatch on the same service. service .dispatch("counter.create", json!({ "id": "c2" }), Session::new()) - .unwrap(); + .expect("direct dispatch should create c2"); let c1: Counter = service.repo().get("c1").unwrap().unwrap(); let c2: Counter = service.repo().get("c2").unwrap().unwrap(); assert_eq!(c1.value, 0); assert_eq!(c2.value, 0); - - let stats = handle.stop().expect("transport should stop cleanly"); - assert_eq!(stats.handled, 1); } -#[test] -fn metadata_becomes_session() { - let bus = Bus::from_queue(InMemoryQueue::new()); +#[tokio::test] +async fn metadata_becomes_session() { + let bus = InMemoryBus::new(); let service = counter_service(); - let handle = microsvc::listen( + // `whoami` reads `ctx.user_id()`, which the runner derives from the message + // metadata (`message_to_session` lowercases keys into session variables). + bus.send_message(command("whoami", "cmd-1", "{}").with_metadata("x-hasura-user-id", "user-42")) + .await + .expect("whoami should enqueue"); + + // Stop on permanent failure so a missing session user would surface as Err; + // `whoami` succeeding proves the metadata became the session. + bus.listen( service.clone(), - "commands", - bus.subscriber().clone(), - Duration::from_millis(10), + RunOptions::idempotent().with_failure_policy(FailurePolicy::Stop), + ) + .await + .expect("whoami should succeed because metadata became the session"); + + // Negative control: without metadata, whoami has no user (permanent + // Unauthorized) and the Stop policy surfaces the failure. + bus.send_message(command("whoami", "cmd-2", "{}")) + .await + .expect("whoami should enqueue"); + let result = bus + .listen( + service.clone(), + RunOptions::idempotent().with_failure_policy(FailurePolicy::Stop), + ) + .await; + assert!( + result.is_err(), + "without metadata, whoami has no session user and must fail" ); - - let event = Event::with_string_payload("cmd-1", "whoami", "{}") - .with_metadata("x-hasura-user-id", "user-42"); - bus.send("commands", event).unwrap(); - - thread::sleep(Duration::from_millis(200)); - - let stats = handle.stop().expect("transport should stop cleanly"); - assert_eq!(stats.handled, 1); - assert_eq!(stats.failed, 0); } -#[test] -fn multiple_services_on_different_queues() { - let bus = Bus::from_queue(InMemoryQueue::new()); +#[tokio::test] +async fn multiple_services_on_different_queues() { + let bus = InMemoryBus::new(); let store = HashMapRepository::new(); let service_a = Arc::new(sourced_rust::register_handlers!( Service::with_repo(store.clone().queued().aggregate::()), command handlers::counter_create, )); - let service_b = Arc::new(sourced_rust::register_handlers!( Service::with_repo(store.queued().aggregate::()), command handlers::counter_increment, )); - let handle_a = microsvc::listen( - service_a.clone(), - "creates", - bus.subscriber().clone(), - Duration::from_millis(10), - ); - let handle_b = microsvc::listen( - service_b.clone(), - "increments", - bus.subscriber().clone(), - Duration::from_millis(10), - ); - - bus.send( - "creates", - Event::with_string_payload("cmd-1", "counter.create", r#"{"id":"c1"}"#), - ) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); - - bus.send( - "increments", - Event::with_string_payload("cmd-2", "counter.increment", r#"{"id":"c1","amount":42}"#), - ) - .unwrap(); - - thread::sleep(Duration::from_millis(200)); + // Each service drains only its own command queue from the shared bus, so the + // two never compete: service_a takes `counter.create`, service_b takes + // `counter.increment`. + bus.send_message(command("counter.create", "cmd-1", r#"{"id":"c1"}"#)) + .await + .expect("create should enqueue"); + bus.listen(service_a.clone(), RunOptions::idempotent()) + .await + .expect("service A should drain the create queue"); + + bus.send_message(command( + "counter.increment", + "cmd-2", + r#"{"id":"c1","amount":42}"#, + )) + .await + .expect("increment should enqueue"); + bus.listen(service_b.clone(), RunOptions::idempotent()) + .await + .expect("service B should drain the increment queue"); let counter: Counter = service_a.repo().get("c1").unwrap().unwrap(); assert_eq!(counter.value, 42); - - let stats_a = handle_a.stop().expect("transport A should stop cleanly"); - let stats_b = handle_b.stop().expect("transport B should stop cleanly"); - assert_eq!(stats_a.handled, 1); - assert_eq!(stats_b.handled, 1); } From a87edcf0c695f6dce2e678b4c9040af0378a88b7 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 17:13:58 -0500 Subject: [PATCH 31/39] test(cutover): migrate microsvc_saga distributed test onto the async InMemoryBus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the threaded legacy-bus choreography (InMemoryQueue + OutboxWorkerThread::spawn_routed + microsvc::listen per service + sleep-poll) with a deterministic, thread-free drive over the async InMemoryBus: - `publish_pending_outbox` claims each service's outbox and forwards messages by destination — worker-addressed messages are point-to-point commands (send_message → consumed via `listen`), saga-addressed messages are events (publish_message → consumed via `subscribe`). - Each round uses a FRESH bus (the in-memory topic log is retained across reads, so a shared bus would re-deliver every prior event to the saga), forwards the pending outbox backlog, then drains the consumers. The loop ends when no service has pending work — i.e. the saga reached Completed. The `stats.handled` assertions (no async analogue) are dropped in favor of the existing domain assertions (saga/order Completed, inventory 95 available / 5 reserved, payment successful). Test 1 (saga_orchestrated) was already bus-free and is unchanged. Both tests pass. Part of [[tasks/async-only-consolidation]] (Phase 1). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/sagas/microsvc_saga.rs | 194 ++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 96 deletions(-) diff --git a/tests/sagas/microsvc_saga.rs b/tests/sagas/microsvc_saga.rs index 307be7e57..dabd5630f 100644 --- a/tests/sagas/microsvc_saga.rs +++ b/tests/sagas/microsvc_saga.rs @@ -9,26 +9,27 @@ //! //! Two tests: //! 1. **Orchestrated** — test runner dispatches commands to each service -//! 2. **Distributed** — services communicate via bus with `microsvc::listen` +//! 2. **Distributed** — services communicate over the async `InMemoryBus` use std::sync::Arc; -use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use serde_json::json; -use sourced_rust::microsvc::{self, Service, Session}; +use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; +use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; use sourced_rust::{ - AggregateBuilder, HashMapRepository, InMemoryQueue, OutboxWorkerThread, Queueable, + AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, + OutboxClaimRef, Queueable, }; use super::handlers; use super::order::{Inventory, Order, OrderFulfillmentSaga, OrderStatus, Payment, SagaStatus}; -fn event_message(name: &str, input: serde_json::Value) -> microsvc::Message { - microsvc::Message::new( +fn event_message(name: &str, input: serde_json::Value) -> Message { + Message::new( name, - microsvc::MessageKind::Event, + MessageKind::Event, serde_json::to_vec(&input).unwrap(), ) } @@ -208,43 +209,67 @@ fn saga_orchestrated() { } // ============================================================================ -// Test 2: Distributed — services communicate via bus transport +// Test 2: Distributed — services communicate over the async InMemoryBus // ============================================================================ -/// Each service runs on its own named queue with an outbox worker routing -/// messages. The test dispatches `StartSaga` and then polls for completion. +/// Drain a service's outbox onto the bus, routing by destination: messages +/// addressed to a worker service (`destination != "saga"`) are point-to-point +/// commands (consumed via `listen`); messages addressed to the saga +/// (`destination == "saga"`) are events (consumed via `subscribe`). Returns how +/// many messages were forwarded. +async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) -> usize { + let claimed = outbox + .claim_async(ClaimOutboxMessages::new( + "saga-outbox-bridge", + 64, + Duration::from_secs(60), + )) + .await + .expect("outbox claim should succeed"); + let count = claimed.len(); + for message in claimed { + let is_event = message.destination.as_deref() == Some("saga"); + let kind = if is_event { + MessageKind::Event + } else { + MessageKind::Command + }; + let bus_message = Message::new(message.event_type.clone(), kind, message.payload.clone()) + .with_id(message.id().to_string()); + if is_event { + bus.publish_message(bus_message) + .await + .expect("saga event should publish"); + } else { + bus.send_message(bus_message) + .await + .expect("service command should enqueue"); + } + let claim = OutboxClaimRef::from_message(&message).expect("claimed message yields a ref"); + outbox + .complete_async(&claim) + .await + .expect("forwarded message should complete"); + } + count +} + +/// Each service owns its aggregate + outbox; the choreography flows entirely +/// over the bus. Instead of threaded outbox workers + long-poll listeners, the +/// test drives the flow deterministically: each round forwards every pending +/// outbox message onto a fresh bus, then drains the consumers (the saga +/// `subscribe`s to events, the worker services `listen` for commands). The loop +/// ends when no service has pending outbox work — i.e. the saga is complete. /// /// ```text -/// ┌──────────────────────────────────────────────────────────────┐ -/// │ Shared Queue (InMemoryQueue) │ -/// │ "saga" "orders" "inventory" "payments" │ -/// └──────────────────────────────────────────────────────────────┘ -/// ↑↓ ↑↓ ↑↓ ↑↓ -/// ┌──────────┐ ┌──────────┐ ┌──────────────┐ ┌──────────┐ -/// │ Saga │ │ Order │ │ Inventory │ │ Payment │ -/// │ Service │ │ Service │ │ Service │ │ Service │ -/// └──────────┘ └──────────┘ └──────────────┘ └──────────┘ +/// StartSaga ─▶ CreateOrder ─▶ OrderCreated ─▶ ReserveInventory ─▶ InventoryReserved +/// ─▶ ProcessPayment ─▶ PaymentSucceeded ─▶ CompleteOrder ─▶ OrderCompleted ─▶ done /// ``` -/// -/// Flow: -/// 1. Saga starts → sends CreateOrder to "orders" -/// 2. Order creates → sends OrderCreated to "saga" -/// 3. Saga → sends ReserveInventory to "inventory" -/// 4. Inventory reserves → sends InventoryReserved to "saga" -/// 5. Saga → sends ProcessPayment to "payments" -/// 6. Payment captures → sends PaymentSucceeded to "saga" -/// 7. Saga → sends CompleteOrder to "orders" -/// 8. Order completes → sends OrderCompleted to "saga" -/// 9. Saga completes -#[test] -fn saga_distributed() { - let queue = InMemoryQueue::new(); - let poll = Duration::from_millis(10); - +#[tokio::test] +async fn saga_distributed() { // === SAGA SERVICE === let saga_repo = HashMapRepository::new(); - let saga_worker = - OutboxWorkerThread::spawn_routed(saga_repo.outbox_store(), queue.clone(), poll); + let saga_outbox = saga_repo.outbox_store(); let saga_svc = Arc::new(sourced_rust::register_handlers!( Service::with_repo(saga_repo.queued().aggregate::()), command handlers::saga::start, @@ -253,49 +278,38 @@ fn saga_distributed() { event handlers::saga::on_payment_succeeded, event handlers::saga::on_order_completed, )); - let saga_listen = microsvc::listen(saga_svc.clone(), "saga", queue.clone(), poll); // === ORDER SERVICE === let order_repo = HashMapRepository::new(); - let order_worker = - OutboxWorkerThread::spawn_routed(order_repo.outbox_store(), queue.clone(), poll); + let order_outbox = order_repo.outbox_store(); let order_svc = Arc::new(sourced_rust::register_handlers!( Service::with_repo(order_repo.queued().aggregate::()), command handlers::orders::create, command handlers::orders::complete, )); - let order_listen = microsvc::listen(order_svc.clone(), "orders", queue.clone(), poll); - // === INVENTORY SERVICE === + // === INVENTORY SERVICE (pre-seeded) === let inventory_repo = HashMapRepository::new(); - let inventory_worker = - OutboxWorkerThread::spawn_routed(inventory_repo.outbox_store(), queue.clone(), poll); - - // Pre-seed inventory before starting the service + let inventory_outbox = inventory_repo.outbox_store(); { let tmp = inventory_repo.clone().aggregate::(); let mut inv = Inventory::new(); inv.initialize("WIDGET-001".to_string(), 100).unwrap(); tmp.commit(&mut inv).unwrap(); } - let inventory_svc = Arc::new(sourced_rust::register_handlers!( Service::with_repo(inventory_repo.queued().aggregate::()), command handlers::inventory::init, command handlers::inventory::reserve, )); - let inventory_listen = - microsvc::listen(inventory_svc.clone(), "inventory", queue.clone(), poll); // === PAYMENT SERVICE === let payment_repo = HashMapRepository::new(); - let payment_worker = - OutboxWorkerThread::spawn_routed(payment_repo.outbox_store(), queue.clone(), poll); + let payment_outbox = payment_repo.outbox_store(); let payment_svc = Arc::new(sourced_rust::register_handlers!( Service::with_repo(payment_repo.queued().aggregate::()), command handlers::payments::process, )); - let payment_listen = microsvc::listen(payment_svc.clone(), "payments", queue.clone(), poll); // === START THE SAGA === saga_svc @@ -312,47 +326,39 @@ fn saga_distributed() { ) .unwrap(); - // === POLL FOR COMPLETION === - let deadline = Instant::now() + Duration::from_secs(10); - loop { - if let Some(saga) = saga_svc.repo().peek("saga-001").unwrap() { - if saga.is_complete() { - break; - } + // === DRIVE THE CHOREOGRAPHY OVER THE BUS UNTIL QUIESCENT === + let mut reached_quiescence = false; + for _ in 0..30 { + // A fresh bus per round bounds delivery to this hop's messages (the + // in-memory topic log is retained across reads, so reusing one bus would + // re-deliver every prior event to the saga). + let bus = InMemoryBus::new(); + let published = publish_pending_outbox(&saga_outbox, &bus).await + + publish_pending_outbox(&order_outbox, &bus).await + + publish_pending_outbox(&inventory_outbox, &bus).await + + publish_pending_outbox(&payment_outbox, &bus).await; + if published == 0 { + reached_quiescence = true; + break; } - assert!( - Instant::now() < deadline, - "Saga should complete within 10 seconds" - ); - thread::sleep(Duration::from_millis(50)); - } - // === STOP TRANSPORTS AND WORKERS === - let saga_stats = saga_listen - .stop() - .expect("saga listener should stop cleanly"); - let order_stats = order_listen - .stop() - .expect("order listener should stop cleanly"); - let _inventory_stats = inventory_listen - .stop() - .expect("inventory listener should stop cleanly"); - let _payment_stats = payment_listen - .stop() - .expect("payment listener should stop cleanly"); - - saga_worker - .stop() - .expect("outbox worker should stop cleanly"); - order_worker - .stop() - .expect("outbox worker should stop cleanly"); - inventory_worker - .stop() - .expect("outbox worker should stop cleanly"); - payment_worker - .stop() - .expect("outbox worker should stop cleanly"); + bus.subscribe(saga_svc.clone(), RunOptions::idempotent()) + .await + .expect("saga should drain its events"); + bus.listen(order_svc.clone(), RunOptions::idempotent()) + .await + .expect("order service should drain its commands"); + bus.listen(inventory_svc.clone(), RunOptions::idempotent()) + .await + .expect("inventory service should drain its commands"); + bus.listen(payment_svc.clone(), RunOptions::idempotent()) + .await + .expect("payment service should drain its commands"); + } + assert!( + reached_quiescence, + "the saga choreography should reach quiescence within the round budget" + ); // === VERIFY FINAL STATE — typed repos return aggregates directly === @@ -369,8 +375,4 @@ fn saga_distributed() { let payment = payment_svc.repo().peek("pay-order-001").unwrap().unwrap(); assert!(payment.is_successful()); - - // Transport stats: saga handled 4 events, orders handled 2 - assert_eq!(saga_stats.handled, 4); - assert_eq!(order_stats.handled, 2); } From cf18edaffca285b213efb7a348089641e876903d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 17:19:10 -0500 Subject: [PATCH 32/39] test(cutover): remove superseded raw-legacy-bus saga tests (distributed.rs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/sagas/distributed.rs drove the order-fulfillment saga over the raw `bus::Bus`/`Subscribable` API with hand-spawned threads and manual aggregate handling (bus.subscribe(&[names]) -> events.recv() loops). The async InMemoryBus has no raw-receiver equivalent — listen/subscribe are Service-driven — so the file cannot be faithfully migrated; a rewrite would duplicate the async microsvc_saga::saga_distributed test (same saga) plus the matrix metadata coverage. Removed as superseded (owner-confirmed): no coverage is lost. Also drops the now-unused event payloads in tests/sagas/order/events.rs (only distributed.rs constructed them). sagas crate: 7 passed, no warnings. Part of [[tasks/async-only-consolidation]] (Phase 1). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/sagas/distributed.rs | 999 ------------------------------------ tests/sagas/main.rs | 1 - tests/sagas/order/events.rs | 47 -- tests/sagas/order/mod.rs | 5 - 4 files changed, 1052 deletions(-) delete mode 100644 tests/sagas/distributed.rs delete mode 100644 tests/sagas/order/events.rs diff --git a/tests/sagas/distributed.rs b/tests/sagas/distributed.rs deleted file mode 100644 index daaa7a964..000000000 --- a/tests/sagas/distributed.rs +++ /dev/null @@ -1,999 +0,0 @@ -//! Distributed Saga with Outbox Pattern -//! -//! Demonstrates a distributed system where each service runs in its own thread: -//! - Saga orchestrator coordinates the overall flow -//! - Each service has its own repository and outbox worker -//! - Services communicate ONLY via the shared queue -//! - No direct calls between services -//! -//! ```text -//! ┌─────────────────────────────────────────────────────────────┐ -//! │ Shared Queue (InMemoryQueue) │ -//! │ thread-safe via Arc> │ -//! └─────────────────────────────────────────────────────────────┘ -//! ↑↓ ↑↓ ↑↓ ↑↓ -//! ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ -//! │ Saga │ │ Order │ │ Inventory │ │ Payment │ -//! │ Thread │ │ Thread │ │ Thread │ │ Thread │ -//! │ │ │ │ │ │ │ │ -//! │ orchestr. │ │ repo + │ │ repo + │ │ repo + │ -//! │ + repo │ │ worker │ │ worker │ │ worker │ -//! └───────────┘ └───────────┘ └───────────┘ └───────────┘ -//! ``` - -use super::order::{ - Inventory, InventoryReservedPayload, Order, OrderCreatedPayload, - OrderFulfillmentCompletedPayload, OrderFulfillmentSaga, OrderFulfillmentStartedPayload, - OrderItem, Payment, PaymentSucceededPayload, -}; -use sourced_rust::{ - bus::Bus, AggregateBuilder, HashMapRepository, InMemoryQueue, OutboxMessage, - OutboxWorkerThread, Queueable, SyncCommitBuilderExt, SyncOutboxCommitExt, -}; -use std::sync::mpsc::channel; -use std::thread; -use std::time::Duration; - -#[test] -fn distributed_saga_with_threads() { - // Shared queue - all services publish to and subscribe from this - let queue = InMemoryQueue::new(); - - // Channel to signal saga completion - let (complete_tx, complete_rx) = channel::(); - - // ========================================================================= - // SAGA ORCHESTRATOR THREAD - // ========================================================================= - let order_fulfillment_saga_queue = queue.clone(); - let saga_complete_tx = complete_tx.clone(); - let order_fulfillment_saga_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn( - repo.outbox_store(), - order_fulfillment_saga_queue.clone(), - Duration::from_millis(10), - ); - let order_fulfillment_saga_repo = repo.queued().aggregate::(); - - // Create a bus for this service - let bus = Bus::from_queue(order_fulfillment_saga_queue); - - // === Start the saga === - let saga_id = "saga-001".to_string(); - let order_id = "order-001".to_string(); - let items = vec![OrderItem { - sku: "WIDGET-001".to_string(), - quantity: 5, - price_cents: 1000, - }]; - - let mut order_fulfillment_saga = OrderFulfillmentSaga::new(); - order_fulfillment_saga - .start( - saga_id.clone(), - order_id.clone(), - "customer-001".to_string(), - items.clone(), - 5000, - ) - .unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:started", saga_id), - "SagaStarted", - &OrderFulfillmentStartedPayload { - saga_id: saga_id.clone(), - order_id: order_id.clone(), - customer_id: "customer-001".to_string(), - items, - total_cents: 5000, - }, - ) - .unwrap(); - order_fulfillment_saga_repo - .outbox_sync(outbox) - .commit_sync(&mut order_fulfillment_saga) - .unwrap(); - - println!( - "[Saga Orchestrator] Started saga {}, waiting for events...", - saga_id - ); - - // === Subscribe to events that advance saga state === - let events = bus.subscribe(&[ - "OrderCreated", - "InventoryReserved", - "PaymentSucceeded", - "OrderCompleted", - ]); - let deadline = std::time::Instant::now() + Duration::from_secs(10); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = events.recv(100) { - match event.event_type.as_str() { - "OrderCreated" => { - let data: OrderCreatedPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga Orchestrator] Order created, waiting for inventory..."); - } - } - "InventoryReserved" => { - let data: InventoryReservedPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga Orchestrator] Inventory reserved, advancing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.inventory_reserved().unwrap(); - order_fulfillment_saga_repo - .commit(&mut order_fulfillment_saga) - .unwrap(); - } - } - "PaymentSucceeded" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga Orchestrator] Payment succeeded, advancing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.payment_succeeded().unwrap(); - order_fulfillment_saga_repo - .commit(&mut order_fulfillment_saga) - .unwrap(); - } - } - "OrderCompleted" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga Orchestrator] Order completed, completing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.complete().unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:completed", saga_id), - "SagaCompleted", - &OrderFulfillmentCompletedPayload { - saga_id: saga_id.clone(), - order_id: order_id.clone(), - }, - ) - .unwrap(); - order_fulfillment_saga_repo - .outbox_sync(outbox) - .commit_sync(&mut order_fulfillment_saga) - .unwrap(); - - // Wait for outbox worker to publish - thread::sleep(Duration::from_millis(50)); - - println!("[Saga Orchestrator] Saga completed!"); - saga_complete_tx.send(saga_id.clone()).unwrap(); - break; - } - } - _ => unreachable!("Subscribed events are filtered"), - } - } - } - - let final_order_fulfillment_saga = - order_fulfillment_saga_repo.peek(&saga_id).unwrap().unwrap(); - println!( - "[Saga Orchestrator] Final saga status: {:?}", - final_order_fulfillment_saga.status() - ); - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // ORDER SERVICE THREAD - // ========================================================================= - let order_queue = queue.clone(); - let order_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn( - repo.outbox_store(), - order_queue.clone(), - Duration::from_millis(10), - ); - let order_repo = repo.queued().aggregate::(); - - // Create a bus for this service - let bus = Bus::from_queue(order_queue); - - println!("[Order Service] Waiting for SagaStarted..."); - - // Subscribe only to events this service cares about - let events = bus.subscribe(&["SagaStarted", "PaymentSucceeded"]); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - let mut order_id: Option = None; - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = events.recv(100) { - match event.event_type.as_str() { - "SagaStarted" => { - let data: OrderFulfillmentStartedPayload = event.decode().unwrap(); - println!("[Order Service] Received SagaStarted, creating order..."); - - let mut order = Order::new(); - order - .create( - data.order_id.clone(), - data.customer_id.clone(), - data.items.clone(), - ) - .unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:created", data.order_id), - "OrderCreated", - &OrderCreatedPayload { - order_id: data.order_id.clone(), - customer_id: data.customer_id, - items: data.items, - total_cents: data.total_cents, - }, - ) - .unwrap(); - order_repo - .outbox_sync(outbox) - .commit_sync(&mut order) - .unwrap(); - - println!("[Order Service] Created order {}", data.order_id); - order_id = Some(data.order_id); - - thread::sleep(Duration::from_millis(50)); - } - "PaymentSucceeded" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if Some(&data.order_id) == order_id.as_ref() { - println!( - "[Order Service] Received PaymentSucceeded, completing order..." - ); - - let mut order = order_repo.get(&data.order_id).unwrap().unwrap(); - order.mark_inventory_reserved().unwrap(); - order.mark_payment_processed().unwrap(); - order.complete().unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:completed", data.order_id), - "OrderCompleted", - &data, - ) - .unwrap(); - order_repo - .outbox_sync(outbox) - .commit_sync(&mut order) - .unwrap(); - - println!("[Order Service] Order completed!"); - thread::sleep(Duration::from_millis(50)); - break; - } - } - _ => unreachable!("Subscribed events are filtered"), - } - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // INVENTORY SERVICE THREAD - // ========================================================================= - let inventory_queue = queue.clone(); - let inventory_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn( - repo.outbox_store(), - inventory_queue.clone(), - Duration::from_millis(10), - ); - let inventory_repo = repo.queued().aggregate::(); - - // Create a bus for this service - let bus = Bus::from_queue(inventory_queue); - - // Initialize inventory first - let mut inv = Inventory::new(); - inv.initialize("WIDGET-001".to_string(), 100).unwrap(); - inventory_repo.commit(&mut inv).unwrap(); - - println!( - "[Inventory Service] Initialized with 100 WIDGET-001, waiting for OrderCreated..." - ); - - // Subscribe only to OrderCreated events - let events = bus.subscribe(&["OrderCreated"]); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = events.recv(100) { - let data: OrderCreatedPayload = event.decode().unwrap(); - println!("[Inventory Service] Received OrderCreated, reserving inventory..."); - - let item = &data.items[0]; - let mut inv = inventory_repo.get(&item.sku).unwrap().unwrap(); - - if inv.can_reserve(item.quantity) { - inv.reserve(data.order_id.clone(), item.quantity).unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:reserved", data.order_id), - "InventoryReserved", - &InventoryReservedPayload { - order_id: data.order_id.clone(), - sku: item.sku.clone(), - quantity: item.quantity, - }, - ) - .unwrap(); - inventory_repo - .outbox_sync(outbox) - .commit_sync(&mut inv) - .unwrap(); - - println!("[Inventory Service] Reserved {} units", item.quantity); - thread::sleep(Duration::from_millis(50)); - } - break; - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // PAYMENT SERVICE THREAD - // ========================================================================= - let payment_queue = queue.clone(); - let payment_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn( - repo.outbox_store(), - payment_queue.clone(), - Duration::from_millis(10), - ); - let payment_repo = repo.queued().aggregate::(); - - // Create a bus for this service - let bus = Bus::from_queue(payment_queue); - - println!("[Payment Service] Waiting for InventoryReserved..."); - - // Subscribe only to InventoryReserved events - let events = bus.subscribe(&["InventoryReserved"]); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = events.recv(100) { - let data: InventoryReservedPayload = event.decode().unwrap(); - println!("[Payment Service] Received InventoryReserved, processing payment..."); - - let mut payment = Payment::new(); - let payment_id = format!("pay-{}", data.order_id); - payment - .initiate(payment_id.clone(), data.order_id.clone(), 5000) - .unwrap(); - payment.authorize("txn-123".to_string()).unwrap(); - payment.capture().unwrap(); - - let outbox = OutboxMessage::encode( - format!("{}:paid", data.order_id), - "PaymentSucceeded", - &PaymentSucceededPayload { - order_id: data.order_id.clone(), - payment_id, - }, - ) - .unwrap(); - payment_repo - .outbox_sync(outbox) - .commit_sync(&mut payment) - .unwrap(); - - println!("[Payment Service] Payment succeeded!"); - thread::sleep(Duration::from_millis(50)); - break; - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // WAIT FOR SAGA COMPLETION - // ========================================================================= - - let completed_saga = complete_rx - .recv_timeout(Duration::from_secs(10)) - .expect("Saga should complete within 10 seconds"); - - assert_eq!(completed_saga, "saga-001"); - - // Join all threads - order_fulfillment_saga_thread - .join() - .expect("Saga thread panicked"); - order_thread.join().expect("Order thread panicked"); - inventory_thread.join().expect("Inventory thread panicked"); - payment_thread.join().expect("Payment thread panicked"); - - // Verify the event flow - let event_types = queue.event_types(); - println!("\nEvent flow: {:?}", event_types); - - assert!(event_types.contains(&"SagaStarted".to_string())); - assert!(event_types.contains(&"OrderCreated".to_string())); - assert!(event_types.contains(&"InventoryReserved".to_string())); - assert!(event_types.contains(&"PaymentSucceeded".to_string())); - assert!(event_types.contains(&"OrderCompleted".to_string())); - assert!(event_types.contains(&"SagaCompleted".to_string())); -} - -/// Distributed saga using point-to-point messaging (send/listen). -/// -/// Same flow as `distributed_saga_with_threads`, but uses named queues -/// instead of fan-out pub/sub. Each service has its own queue: -/// -/// ```text -/// ┌─────────────────────────────────────────────────────────────┐ -/// │ Shared Queue (InMemoryQueue) │ -/// │ named queues via send/listen (point-to-point) │ -/// └─────────────────────────────────────────────────────────────┘ -/// "saga" "orders" "inventory" "payments" -/// ↑↓ ↑↓ ↑↓ ↑↓ -/// ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ -/// │ Saga │ │ Order │ │ Inventory │ │ Payment │ -/// │ Thread │ │ Thread │ │ Thread │ │ Thread │ -/// └───────────┘ └───────────┘ └───────────┘ └───────────┘ -/// ``` -/// -/// The outbox worker uses `spawn_routed` which checks `msg.destination` -/// and routes via `send(queue, event)` instead of `publish(event)`. -#[test] -fn distributed_saga_with_send_listen() { - // Shared queue - all services send to named queues within this - let queue = InMemoryQueue::new(); - - // Channel to signal saga completion - let (complete_tx, complete_rx) = channel::(); - - // ========================================================================= - // SAGA ORCHESTRATOR THREAD - // ========================================================================= - let order_fulfillment_saga_queue = queue.clone(); - let saga_complete_tx = complete_tx.clone(); - let order_fulfillment_saga_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - // spawn_routed: checks msg.destination → send() if set, publish() if not - let worker = OutboxWorkerThread::spawn_routed( - repo.outbox_store(), - order_fulfillment_saga_queue.clone(), - Duration::from_millis(10), - ); - let order_fulfillment_saga_repo = repo.queued().aggregate::(); - - let bus = Bus::from_queue(order_fulfillment_saga_queue); - - // === Start the saga === - let saga_id = "saga-001".to_string(); - let order_id = "order-001".to_string(); - let items = vec![OrderItem { - sku: "WIDGET-001".to_string(), - quantity: 5, - price_cents: 1000, - }]; - - let mut order_fulfillment_saga = OrderFulfillmentSaga::new(); - order_fulfillment_saga - .start( - saga_id.clone(), - order_id.clone(), - "customer-001".to_string(), - items.clone(), - 5000, - ) - .unwrap(); - - // Send to the "orders" queue (point-to-point) - let outbox = OutboxMessage::encode_to( - format!("{}:started", saga_id), - "SagaStarted", - "orders", // destination queue - &OrderFulfillmentStartedPayload { - saga_id: saga_id.clone(), - order_id: order_id.clone(), - customer_id: "customer-001".to_string(), - items, - total_cents: 5000, - }, - ) - .unwrap(); - order_fulfillment_saga_repo - .outbox_sync(outbox) - .commit_sync(&mut order_fulfillment_saga) - .unwrap(); - - println!( - "[Saga/SendListen] Started saga {}, listening on 'saga' queue...", - saga_id - ); - - // === Listen on the "saga" queue for events that advance saga state === - let deadline = std::time::Instant::now() + Duration::from_secs(10); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = bus.listen("saga", 100) { - match event.event_type.as_str() { - "OrderCreated" => { - let data: OrderCreatedPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga/SendListen] Order created, waiting for inventory..."); - } - } - "InventoryReserved" => { - let data: InventoryReservedPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga/SendListen] Inventory reserved, advancing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.inventory_reserved().unwrap(); - order_fulfillment_saga_repo - .commit(&mut order_fulfillment_saga) - .unwrap(); - } - } - "PaymentSucceeded" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga/SendListen] Payment succeeded, advancing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.payment_succeeded().unwrap(); - order_fulfillment_saga_repo - .commit(&mut order_fulfillment_saga) - .unwrap(); - } - } - "OrderCompleted" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if data.order_id == order_id { - println!("[Saga/SendListen] Order completed, completing saga..."); - let mut order_fulfillment_saga = - order_fulfillment_saga_repo.get(&saga_id).unwrap().unwrap(); - order_fulfillment_saga.complete().unwrap(); - - // SagaCompleted has no specific destination, but we can - // still route it to a queue (or use publish for fan-out) - let outbox = OutboxMessage::encode_to( - format!("{}:completed", saga_id), - "SagaCompleted", - "saga-completed", // destination queue - &OrderFulfillmentCompletedPayload { - saga_id: saga_id.clone(), - order_id: order_id.clone(), - }, - ) - .unwrap(); - order_fulfillment_saga_repo - .outbox_sync(outbox) - .commit_sync(&mut order_fulfillment_saga) - .unwrap(); - - thread::sleep(Duration::from_millis(50)); - - println!("[Saga/SendListen] Saga completed!"); - saga_complete_tx.send(saga_id.clone()).unwrap(); - break; - } - } - other => { - println!("[Saga/SendListen] Unexpected event: {}", other); - } - } - } - } - - let final_order_fulfillment_saga = - order_fulfillment_saga_repo.peek(&saga_id).unwrap().unwrap(); - println!( - "[Saga/SendListen] Final saga status: {:?}", - final_order_fulfillment_saga.status() - ); - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // ORDER SERVICE THREAD - // ========================================================================= - let order_queue = queue.clone(); - let order_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn_routed( - repo.outbox_store(), - order_queue.clone(), - Duration::from_millis(10), - ); - let order_repo = repo.clone().queued().aggregate::(); - - let bus = Bus::from_queue(order_queue); - - println!("[Order/SendListen] Listening on 'orders' queue..."); - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - let mut order_id: Option = None; - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = bus.listen("orders", 100) { - match event.event_type.as_str() { - "SagaStarted" => { - let data: OrderFulfillmentStartedPayload = event.decode().unwrap(); - println!("[Order/SendListen] Received SagaStarted, creating order..."); - - let mut order = Order::new(); - order - .create( - data.order_id.clone(), - data.customer_id.clone(), - data.items.clone(), - ) - .unwrap(); - - // Send OrderCreated to both "saga" and "inventory" queues - let payload = OrderCreatedPayload { - order_id: data.order_id.clone(), - customer_id: data.customer_id, - items: data.items, - total_cents: data.total_cents, - }; - - let outbox_saga = OutboxMessage::encode_to( - format!("{}:created:saga", data.order_id), - "OrderCreated", - "saga", - &payload, - ) - .unwrap(); - - let outbox_inventory = OutboxMessage::encode_to( - format!("{}:created:inventory", data.order_id), - "OrderCreated", - "inventory", - &payload, - ) - .unwrap(); - - repo.outbox_sync(outbox_saga) - .outbox_sync(outbox_inventory) - .commit_sync(&mut order) - .unwrap(); - - println!("[Order/SendListen] Created order {}", data.order_id); - order_id = Some(data.order_id); - - thread::sleep(Duration::from_millis(50)); - } - "PaymentSucceeded" => { - let data: PaymentSucceededPayload = event.decode().unwrap(); - if Some(&data.order_id) == order_id.as_ref() { - println!( - "[Order/SendListen] Received PaymentSucceeded, completing order..." - ); - - let mut order = order_repo.get(&data.order_id).unwrap().unwrap(); - order.mark_inventory_reserved().unwrap(); - order.mark_payment_processed().unwrap(); - order.complete().unwrap(); - - let outbox = OutboxMessage::encode_to( - format!("{}:completed", data.order_id), - "OrderCompleted", - "saga", - &data, - ) - .unwrap(); - order_repo - .outbox_sync(outbox) - .commit_sync(&mut order) - .unwrap(); - - println!("[Order/SendListen] Order completed!"); - thread::sleep(Duration::from_millis(50)); - break; - } - } - other => { - println!("[Order/SendListen] Unexpected event: {}", other); - } - } - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // INVENTORY SERVICE THREAD - // ========================================================================= - let inventory_queue = queue.clone(); - let inventory_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn_routed( - repo.outbox_store(), - inventory_queue.clone(), - Duration::from_millis(10), - ); - let inventory_repo = repo.clone().queued().aggregate::(); - - let bus = Bus::from_queue(inventory_queue); - - // Initialize inventory - let mut inv = Inventory::new(); - inv.initialize("WIDGET-001".to_string(), 100).unwrap(); - inventory_repo.commit(&mut inv).unwrap(); - - println!( - "[Inventory/SendListen] Initialized with 100 WIDGET-001, listening on 'inventory' queue..." - ); - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = bus.listen("inventory", 100) { - let data: OrderCreatedPayload = event.decode().unwrap(); - println!("[Inventory/SendListen] Received OrderCreated, reserving inventory..."); - - let item = &data.items[0]; - let mut inv = inventory_repo.get(&item.sku).unwrap().unwrap(); - - if inv.can_reserve(item.quantity) { - inv.reserve(data.order_id.clone(), item.quantity).unwrap(); - - // Send InventoryReserved to both "saga" and "payments" queues - let payload = InventoryReservedPayload { - order_id: data.order_id.clone(), - sku: item.sku.clone(), - quantity: item.quantity, - }; - - let outbox_saga = OutboxMessage::encode_to( - format!("{}:reserved:saga", data.order_id), - "InventoryReserved", - "saga", - &payload, - ) - .unwrap(); - - let outbox_payments = OutboxMessage::encode_to( - format!("{}:reserved:payments", data.order_id), - "InventoryReserved", - "payments", - &payload, - ) - .unwrap(); - - repo.outbox_sync(outbox_saga) - .outbox_sync(outbox_payments) - .commit_sync(&mut inv) - .unwrap(); - - println!("[Inventory/SendListen] Reserved {} units", item.quantity); - thread::sleep(Duration::from_millis(50)); - } - break; - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // PAYMENT SERVICE THREAD - // ========================================================================= - let payment_queue = queue.clone(); - let payment_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn_routed( - repo.outbox_store(), - payment_queue.clone(), - Duration::from_millis(10), - ); - let _payment_repo = repo.clone().queued().aggregate::(); - - let bus = Bus::from_queue(payment_queue); - - println!("[Payment/SendListen] Listening on 'payments' queue..."); - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = bus.listen("payments", 100) { - let data: InventoryReservedPayload = event.decode().unwrap(); - println!("[Payment/SendListen] Received InventoryReserved, processing payment..."); - - let mut payment = Payment::new(); - let payment_id = format!("pay-{}", data.order_id); - payment - .initiate(payment_id.clone(), data.order_id.clone(), 5000) - .unwrap(); - payment.authorize("txn-123".to_string()).unwrap(); - payment.capture().unwrap(); - - // Send PaymentSucceeded to both "saga" and "orders" queues - let payload = PaymentSucceededPayload { - order_id: data.order_id.clone(), - payment_id, - }; - - let outbox_saga = OutboxMessage::encode_to( - format!("{}:paid:saga", data.order_id), - "PaymentSucceeded", - "saga", - &payload, - ) - .unwrap(); - - let outbox_orders = OutboxMessage::encode_to( - format!("{}:paid:orders", data.order_id), - "PaymentSucceeded", - "orders", - &payload, - ) - .unwrap(); - - repo.outbox_sync(outbox_saga) - .outbox_sync(outbox_orders) - .commit_sync(&mut payment) - .unwrap(); - - println!("[Payment/SendListen] Payment succeeded!"); - thread::sleep(Duration::from_millis(50)); - break; - } - } - - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // WAIT FOR SAGA COMPLETION - // ========================================================================= - - let completed_saga = complete_rx - .recv_timeout(Duration::from_secs(10)) - .expect("Saga should complete within 10 seconds"); - - assert_eq!(completed_saga, "saga-001"); - - // Join all threads - order_fulfillment_saga_thread - .join() - .expect("Saga thread panicked"); - order_thread.join().expect("Order thread panicked"); - inventory_thread.join().expect("Inventory thread panicked"); - payment_thread.join().expect("Payment thread panicked"); - - // With send/listen, events go to named queues, not the fan-out log. - // Verify that the fan-out log is empty (no publish calls were made). - let event_types = queue.event_types(); - assert!( - event_types.is_empty(), - "Fan-out log should be empty when using send/listen, got: {:?}", - event_types - ); -} - -/// Metadata flows end-to-end: Entity → EventRecord → OutboxMessage → bus::Event → subscriber. -/// -/// A producer thread creates an order with correlation/causation metadata, -/// publishes via OutboxWorkerThread, and a subscriber thread verifies the -/// metadata arrives on the received bus::Event. -#[test] -fn metadata_propagates_across_bus_to_subscriber() { - let queue = InMemoryQueue::new(); - let (result_tx, result_rx) = channel::<(Option, Option, Option)>(); - - // ========================================================================= - // SUBSCRIBER THREAD — receives the event and checks metadata - // ========================================================================= - let subscriber_queue = queue.clone(); - let subscriber_thread = thread::spawn(move || { - let bus = Bus::from_queue(subscriber_queue); - let events = bus.subscribe(&["OrderCreated"]); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - - while std::time::Instant::now() < deadline { - if let Ok(Some(event)) = events.recv(100) { - // Extract metadata from the received bus::Event - let correlation = event.correlation_id().map(|s| s.to_string()); - let causation = event.causation_id().map(|s| s.to_string()); - let user_id = event.meta("user_id").map(|s| s.to_string()); - result_tx.send((correlation, causation, user_id)).unwrap(); - return; - } - } - panic!("Subscriber timed out waiting for OrderCreated event"); - }); - - // ========================================================================= - // PRODUCER THREAD — creates entity with metadata, publishes via outbox - // ========================================================================= - let producer_queue = queue.clone(); - let producer_thread = thread::spawn(move || { - let repo = HashMapRepository::new(); - let worker = OutboxWorkerThread::spawn( - repo.outbox_store(), - producer_queue, - Duration::from_millis(10), - ); - let order_repo = repo.aggregate::(); - - // Create an order with metadata on the entity - let mut order = Order::new(); - order.entity.set_correlation_id("req-distributed-001"); - order.entity.set_causation_id("cmd-create-order"); - order.entity.set_meta("user_id", "u-99"); - order - .create( - "order-meta-001".to_string(), - "customer-meta-001".to_string(), - vec![OrderItem { - sku: "WIDGET-META".to_string(), - quantity: 1, - price_cents: 500, - }], - ) - .unwrap(); - - // Metadata propagates automatically from entity context - let outbox = OutboxMessage::encode_for_entity( - "order-meta-001:created", - "OrderCreated", - &OrderCreatedPayload { - order_id: "order-meta-001".to_string(), - customer_id: "customer-meta-001".to_string(), - items: vec![OrderItem { - sku: "WIDGET-META".to_string(), - quantity: 1, - price_cents: 500, - }], - total_cents: 500, - }, - &order.entity, - ) - .unwrap(); - - order_repo - .outbox_sync(outbox) - .commit_sync(&mut order) - .unwrap(); - - // Give the outbox worker time to publish - thread::sleep(Duration::from_millis(200)); - worker.stop().expect("outbox worker should stop cleanly"); - }); - - // ========================================================================= - // VERIFY — subscriber received metadata - // ========================================================================= - let (correlation, causation, user_id) = result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("Should receive metadata within 5 seconds"); - - assert_eq!(correlation.as_deref(), Some("req-distributed-001")); - assert_eq!(causation.as_deref(), Some("cmd-create-order")); - assert_eq!(user_id.as_deref(), Some("u-99")); - - producer_thread.join().expect("Producer thread panicked"); - subscriber_thread - .join() - .expect("Subscriber thread panicked"); -} diff --git a/tests/sagas/main.rs b/tests/sagas/main.rs index 1ce2dbd38..e53d3bce9 100644 --- a/tests/sagas/main.rs +++ b/tests/sagas/main.rs @@ -5,7 +5,6 @@ //! business process across multiple aggregates, with compensation //! (rollback) capabilities when steps fail. -mod distributed; mod handlers; mod microsvc_saga; mod orchestration; diff --git a/tests/sagas/order/events.rs b/tests/sagas/order/events.rs deleted file mode 100644 index fb52bba5c..000000000 --- a/tests/sagas/order/events.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! Event payloads for inter-service communication in sagas. - -use bitcode::{Decode, Encode}; -use serde::{Deserialize, Serialize}; - -use super::OrderItem; - -/// Payload for SagaStarted event - initiates the saga. -#[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] -pub struct OrderFulfillmentStartedPayload { - pub saga_id: String, - pub order_id: String, - pub customer_id: String, - pub items: Vec, - pub total_cents: u32, -} - -/// Payload for OrderCreated event. -#[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] -pub struct OrderCreatedPayload { - pub order_id: String, - pub customer_id: String, - pub items: Vec, - pub total_cents: u32, -} - -/// Payload for InventoryReserved event. -#[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] -pub struct InventoryReservedPayload { - pub order_id: String, - pub sku: String, - pub quantity: u32, -} - -/// Payload for PaymentSucceeded event. -#[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] -pub struct PaymentSucceededPayload { - pub order_id: String, - pub payment_id: String, -} - -/// Payload for SagaCompleted event - saga finished successfully. -#[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] -pub struct OrderFulfillmentCompletedPayload { - pub saga_id: String, - pub order_id: String, -} diff --git a/tests/sagas/order/mod.rs b/tests/sagas/order/mod.rs index 9fbdb13c8..11210b5e8 100644 --- a/tests/sagas/order/mod.rs +++ b/tests/sagas/order/mod.rs @@ -1,13 +1,8 @@ -mod events; mod inventory; mod order_aggregate; mod payment; mod saga; -pub use events::{ - InventoryReservedPayload, OrderCreatedPayload, OrderFulfillmentCompletedPayload, - OrderFulfillmentStartedPayload, PaymentSucceededPayload, -}; pub use inventory::Inventory; pub use order_aggregate::{Order, OrderItem, OrderStatus}; pub use payment::{Payment, PaymentStatus}; From c0bd6a5c53daf6fe0c80e319488f4182830d1d27 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 17:38:27 -0500 Subject: [PATCH 33/39] test(cutover): decouple projection handlers from bus::Event; migrate board onto InMemoryBus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The projection handlers of BOTH the gold-standard matrix and the board still decoded via bus::Event (`Event::try_from(ctx.message())` + event.decode/ json_decode + event.id) — so the legacy bus could not be removed without touching the gold-standard test. Decoupled both (refactor, not delete): - decode straight from ctx.message().payload(): serde_json::from_slice for the matrix (JSON), BitcodePayloadCodec::decode for the board (bitcode — identical bytes to the old event.decode()); match on ctx.message().name(); event id from ctx.message().id(). Dropped the bus::Event `event()` helper from both projection handlers/mod.rs. - board main.rs: replaced InMemoryQueue + OutboxWorkerThread + the threaded start_board_projection_service + wait_for_* polling with publish_pending_outbox (fan-out events) + a single bus.subscribe; the projection's monotonic source_version guard makes the per-event-type drain order-independent. Added projections_service::load_board (direct read) replacing the poll loop. matrix: 2 passed (in-memory cell + refactored saga, both exercise the decode); board: 3 passed; sagas: 7 passed. clippy/fmt clean. Part of [[tasks/async-only-consolidation]] (Phase 1 — last test migration before the src removal). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../projection_service/handlers/checkout.rs | 16 ++-- .../projection_service/handlers/mod.rs | 9 +- .../projection_service/handlers/seat.rs | 12 +-- tests/distributed_read_model_board/main.rs | 92 ++++++++++--------- .../projections_service/handlers/board.rs | 31 +++---- .../projections_service/handlers/mod.rs | 9 -- .../projections_service/mod.rs | 56 +++-------- 7 files changed, 88 insertions(+), 137 deletions(-) diff --git a/tests/distributed_read_model/projection_service/handlers/checkout.rs b/tests/distributed_read_model/projection_service/handlers/checkout.rs index 96574a07c..65c2cfc2f 100644 --- a/tests/distributed_read_model/projection_service/handlers/checkout.rs +++ b/tests/distributed_read_model/projection_service/handlers/checkout.rs @@ -19,12 +19,9 @@ pub fn guard(ctx: &Context) -> bool { } pub fn handle(ctx: &Context) -> Result { - let event = super::event(ctx)?; - - match event.event_type.as_str() { + match ctx.message().name() { checkout_event::STARTED => { - let msg: CheckoutStarted = event - .json_decode() + let msg: CheckoutStarted = serde_json::from_slice(ctx.message().payload()) .map_err(|err| HandlerError::DecodeFailed(format!("checkout started: {err}")))?; let checkout = CheckoutView { checkout_id: msg.checkout_id.clone(), @@ -45,9 +42,10 @@ pub fn handle(ctx: &Context) -> Result { - let msg: SeatReservationCompleted = event.json_decode().map_err(|err| { - HandlerError::DecodeFailed(format!("seat reservation completed: {err}")) - })?; + let msg: SeatReservationCompleted = serde_json::from_slice(ctx.message().payload()) + .map_err(|err| { + HandlerError::DecodeFailed(format!("seat reservation completed: {err}")) + })?; let checkout = CheckoutView { checkout_id: msg.checkout_id.clone(), seat_id: msg.seat_id, @@ -73,7 +71,7 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), } - Ok(json!({ "event_id": event.id })) + Ok(json!({ "event_id": ctx.message().id() })) } fn checkout_step(checkout_id: &str, step: &str, detail: &str) -> CheckoutStepView { diff --git a/tests/distributed_read_model/projection_service/handlers/mod.rs b/tests/distributed_read_model/projection_service/handlers/mod.rs index acb8dc8d8..80d6e52cd 100644 --- a/tests/distributed_read_model/projection_service/handlers/mod.rs +++ b/tests/distributed_read_model/projection_service/handlers/mod.rs @@ -4,16 +4,9 @@ pub mod checkout; pub mod seat; -use sourced_rust::bus::Event; -use sourced_rust::microsvc::{Context, HandlerError}; +use sourced_rust::microsvc::HandlerError; use sourced_rust::ReadModelError; -use crate::projection_service::ProjectionDependencies; - -pub fn event(ctx: &Context) -> Result { - Event::try_from(ctx.message()) -} - pub fn read_model_error(err: ReadModelError) -> HandlerError { HandlerError::Repository(err.into()) } diff --git a/tests/distributed_read_model/projection_service/handlers/seat.rs b/tests/distributed_read_model/projection_service/handlers/seat.rs index 015c71edf..128b1aa98 100644 --- a/tests/distributed_read_model/projection_service/handlers/seat.rs +++ b/tests/distributed_read_model/projection_service/handlers/seat.rs @@ -13,12 +13,9 @@ pub fn guard(ctx: &Context) -> bool { } pub fn handle(ctx: &Context) -> Result { - let event = super::event(ctx)?; - - match event.event_type.as_str() { + match ctx.message().name() { seat_event::ADDED => { - let msg: SeatAdded = event - .json_decode() + let msg: SeatAdded = serde_json::from_slice(ctx.message().payload()) .map_err(|err| HandlerError::DecodeFailed(format!("seat added: {err}")))?; let row = SeatView { seat_id: msg.seat_id, @@ -32,8 +29,7 @@ pub fn handle(ctx: &Context) -> Result { - let msg: SeatReserved = event - .json_decode() + let msg: SeatReserved = serde_json::from_slice(ctx.message().payload()) .map_err(|err| HandlerError::DecodeFailed(format!("seat reserved: {err}")))?; let seat = SeatView { seat_id: msg.seat_id.clone(), @@ -55,5 +51,5 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), } - Ok(json!({ "event_id": event.id })) + Ok(json!({ "event_id": ctx.message().id() })) } diff --git a/tests/distributed_read_model_board/main.rs b/tests/distributed_read_model_board/main.rs index 92b115731..af4ea702d 100644 --- a/tests/distributed_read_model_board/main.rs +++ b/tests/distributed_read_model_board/main.rs @@ -1,5 +1,5 @@ //! Distributed read-model example: a kanban board projected into normalized -//! `boards` + `cards` tables. +//! `boards` + `cards` tables, over the async `InMemoryBus`. //! //! - the **board service** owns the `Board` aggregate (cards are aggregate //! state) and its outbox; @@ -14,18 +14,18 @@ mod projections_service; mod query_service; mod read_models; -use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use board_service::{AddCard, MoveCard, OpenBoard, RemoveCard}; -use projections_service::{start_board_projection_service, wait_for_board}; +use projections_service::{load_board, service as build_projection}; use query_service::BoardQueryService; use read_models::register_schemas; use serde::Serialize; -use sourced_rust::microsvc::{Service, Session}; +use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; +use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; use sourced_rust::{ - AggregateBuilder, HashMapRepository, InMemoryQueue, InMemoryReadModelStore, OutboxWorkerThread, - Queueable, + AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, + InMemoryReadModelStore, OutboxClaimRef, Queueable, }; fn dispatch(service: &Service, command: &str, input: C) @@ -42,37 +42,47 @@ where .unwrap_or_else(|err| panic!("{command} should dispatch: {err:?}")); } -fn wait_for_published_events(queue: &InMemoryQueue, expected_count: usize) { - let deadline = Instant::now() + Duration::from_secs(10); - - loop { - if queue.len() >= expected_count { - return; - } - - assert!( - Instant::now() < deadline, - "timed out waiting for outbox worker to publish events" - ); - thread::sleep(Duration::from_millis(10)); +/// Forward the board's outbox events onto the bus. Board events are fan-out +/// (`domain_event`, no destination), so each is published as an event for the +/// projection's `subscribe` to drain. Payload bytes are forwarded verbatim +/// (bitcode), and the projection decodes them with `BitcodePayloadCodec`. +async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) { + let claimed = outbox + .claim_async(ClaimOutboxMessages::new( + "board-outbox-bridge", + 64, + Duration::from_secs(60), + )) + .await + .expect("outbox claim should succeed"); + for message in claimed { + bus.publish_message( + Message::new( + message.event_type.clone(), + MessageKind::Event, + message.payload.clone(), + ) + .with_id(message.id().to_string()), + ) + .await + .expect("board event should publish to the bus"); + let claim = OutboxClaimRef::from_message(&message).expect("claimed message yields a ref"); + outbox + .complete_async(&claim) + .await + .expect("forwarded event should complete"); } } -#[test] -fn board_service_feeds_a_normalized_card_read_model() { - let queue = InMemoryQueue::new(); - +#[tokio::test] +async fn board_service_feeds_a_normalized_card_read_model() { let board_store = HashMapRepository::new(); - let board_service = board_service::model_service(board_store.clone().queued().aggregate()); - let worker = OutboxWorkerThread::spawn( - board_store.outbox_store(), - queue.clone(), - Duration::from_millis(5), - ); + let board_outbox = board_store.outbox_store(); + let board_service = board_service::model_service(board_store.queued().aggregate()); let read_store = InMemoryReadModelStore::new(); register_schemas(&read_store).expect("relational schemas should register"); - let projection = start_board_projection_service(queue.clone(), read_store.clone()); + let projection = build_projection(read_store.clone()); let query_service = BoardQueryService::new(read_store.clone()); dispatch( @@ -125,11 +135,17 @@ fn board_service_feeds_a_normalized_card_read_model() { }, ); - wait_for_published_events(&queue, 5); + // Forward the board's outbox events onto the bus, then drain them into the + // projection in one pass. The projection's monotonic `source_version` guard + // makes the per-event-type drain order-independent (the highest version + // wins), so the final board reflects every processed event. + let bus = InMemoryBus::new(); + publish_pending_outbox(&board_outbox, &bus).await; + bus.subscribe(projection, RunOptions::idempotent()) + .await + .expect("projection should drain the board events"); - let board = wait_for_board(&read_store, "board-1", |board| { - board.cards.len() == 1 && board.cards[0].column == "doing" - }); + let board = load_board(&read_store, "board-1").expect("board should be projected"); assert_eq!(board.name, "Roadmap"); assert_eq!(board.cards.len(), 1, "removed card should be deleted"); @@ -170,10 +186,4 @@ fn board_service_feeds_a_normalized_card_read_model() { .expect("write-side load should succeed") .expect("write-side board should exist"); assert_eq!(write_side.cards.len(), 1); - - projection - .stop() - .expect("projection service should stop cleanly"); - let stats = worker.stop().expect("worker should stop cleanly"); - assert!(stats.messages_published >= 5); } diff --git a/tests/distributed_read_model_board/projections_service/handlers/board.rs b/tests/distributed_read_model_board/projections_service/handlers/board.rs index 266419355..002f163e4 100644 --- a/tests/distributed_read_model_board/projections_service/handlers/board.rs +++ b/tests/distributed_read_model_board/projections_service/handlers/board.rs @@ -4,9 +4,8 @@ //! snapshots under out-of-order delivery. use serde_json::{json, Value}; -use sourced_rust::bus::Event; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::ReadModelWorkspaceExt; +use sourced_rust::{BitcodePayloadCodec, PayloadCodec, ReadModelWorkspaceExt}; use crate::board_service::BoardSnapshot; use crate::projections_service::{read_model_error, ProjectionDependencies}; @@ -24,11 +23,13 @@ pub fn guard(ctx: &Context) -> bool { } pub fn handle(ctx: &Context) -> Result { - let event = super::event(ctx)?; - let snapshot: BoardSnapshot = event - .decode() + let message_id = ctx + .message() + .id() + .ok_or_else(|| HandlerError::DecodeFailed("board projection message has no id".into()))?; + let snapshot: BoardSnapshot = BitcodePayloadCodec::decode(ctx.message().payload()) .map_err(|err| HandlerError::DecodeFailed(format!("board snapshot: {err}")))?; - let version = event_version(&event); + let version = event_version(message_id); let updated_view = updated_board_view(&snapshot, version); let mut workspace = ctx.read_model_store().workspace(); @@ -53,7 +54,7 @@ pub fn handle(ctx: &Context) -> Result BoardView { @@ -83,10 +84,8 @@ fn updated_board_view(snapshot: &BoardSnapshot, version: i64) -> BoardView { /// The aggregate version is the trailing segment of the outbox event id /// (`::`). -fn event_version(event: &Event) -> i64 { - event - .id - .rsplit(':') +fn event_version(id: &str) -> i64 { + id.rsplit(':') .next() .expect("board projection event id should include a version segment") .parse() @@ -99,10 +98,7 @@ mod tests { #[test] fn event_version_parses_trailing_outbox_segment() { - let event = - Event::with_string_payload("board-1:board.card_added:42", "board.card_added", "{}"); - - assert_eq!(event_version(&event), 42); + assert_eq!(event_version("board-1:board.card_added:42"), 42); } #[test] @@ -110,9 +106,6 @@ mod tests { expected = "board projection event id should end with a numeric aggregate version" )] fn event_version_panics_on_malformed_outbox_segment() { - let event = - Event::with_string_payload("board-1:board.card_added:bad", "board.card_added", "{}"); - - event_version(&event); + event_version("board-1:board.card_added:bad"); } } diff --git a/tests/distributed_read_model_board/projections_service/handlers/mod.rs b/tests/distributed_read_model_board/projections_service/handlers/mod.rs index 8c94afc29..da069d4f0 100644 --- a/tests/distributed_read_model_board/projections_service/handlers/mod.rs +++ b/tests/distributed_read_model_board/projections_service/handlers/mod.rs @@ -2,12 +2,3 @@ //! handler that owns the matching read-model rows. pub mod board; - -use sourced_rust::bus::Event; -use sourced_rust::microsvc::{Context, HandlerError}; - -use crate::projections_service::ProjectionDependencies; - -pub fn event(ctx: &Context) -> Result { - Event::try_from(ctx.message()) -} diff --git a/tests/distributed_read_model_board/projections_service/mod.rs b/tests/distributed_read_model_board/projections_service/mod.rs index fe64d4ae4..fa50a107f 100644 --- a/tests/distributed_read_model_board/projections_service/mod.rs +++ b/tests/distributed_read_model_board/projections_service/mod.rs @@ -4,28 +4,14 @@ mod handlers; use std::sync::Arc; -use std::thread; -use std::time::{Duration, Instant}; -use sourced_rust::bus::Subscribable; -use sourced_rust::microsvc::{self, HandlerError, Service}; -use sourced_rust::{InMemoryQueue, InMemoryReadModelStore, ReadModelError, ReadModelWorkspaceExt}; +use sourced_rust::microsvc::{HandlerError, Service}; +use sourced_rust::{InMemoryReadModelStore, ReadModelError, ReadModelWorkspaceExt}; use crate::read_models::{board_key, BoardView}; pub type ProjectionDependencies = InMemoryReadModelStore; -pub fn start_board_projection_service( - queue: InMemoryQueue, - store: InMemoryReadModelStore, -) -> microsvc::TransportHandle { - microsvc::subscribe( - service(store), - queue.new_subscriber(), - Duration::from_millis(10), - ) -} - pub fn service(store: InMemoryReadModelStore) -> Arc> { Arc::new(sourced_rust::register_handlers!( Service::with_read_model_store(store), @@ -37,31 +23,15 @@ fn read_model_error(err: ReadModelError) -> HandlerError { HandlerError::Repository(err.into()) } -pub fn wait_for_board( - store: &InMemoryReadModelStore, - board_id: &str, - ready: impl Fn(&BoardView) -> bool, -) -> BoardView { - let deadline = Instant::now() + Duration::from_secs(10); - - loop { - let mut session = store.workspace(); - if let Some(board) = session - .load::(board_key(board_id)) - .include("cards") - .one() - .expect("board load should succeed") - .map(|view| view.data) - { - if ready(&board) { - return board; - } - } - - assert!( - Instant::now() < deadline, - "timed out waiting for board projection" - ); - thread::sleep(Duration::from_millis(10)); - } +/// Load the projected board (with its cards) from the read store. After the +/// bus has been drained into the projection service, the board reflects every +/// processed event. +pub fn load_board(store: &InMemoryReadModelStore, board_id: &str) -> Option { + store + .workspace() + .load::(board_key(board_id)) + .include("cards") + .one() + .expect("board load should succeed") + .map(|view| view.data) } From a64f92ce6930fdab2b9cfc5b006eacb3665ee1ca Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 17:53:30 -0500 Subject: [PATCH 34/39] refactor!: remove the legacy sync bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy sync bus is fully superseded by the async bus facade (InMemoryBus + the BusConsumer listen/subscribe + OutboxSource) and had no remaining consumers after the test migrations. Removed: - `src/bus/` entirely (Bus/Subscribable/InMemoryQueue/Listener/Sender/EventBus/ Event/Publisher, ~1.4k lines). - `OutboxWorkerThread` + WorkerStats + OutboxWorkerJoinError (the threaded outbox->bus bridge) and `src/outbox_worker/thread.rs`. - The bus-gated `microsvc::service` surface: `dispatch_event`, `dispatch_listened_event`, `subscribe`/`listen`, `TransportHandle` + `TransportStats`/`TransportJoinError`, and the `From<&Event> for Message` / `TryFrom<&Message> for Event` / `from_bus_event` bridges (+ their unit tests). - The `bus` Cargo feature (out of `default`); `http`/`grpc` no longer depend on it — they use the unconditional `microsvc::Message`, confirmed by building `--features http,grpc`. - The bus-gated crate-root re-exports (`InMemoryQueue`, `bus::Message`, the threaded-worker types). All consumers were migrated first (transport_subscribe/listen, microsvc_saga, the board) or removed as superseded (sagas/distributed.rs), and both projection handlers were decoupled from `bus::Event`. Default test sweep: 238 lib + all integration crates green; `--features http,grpc` builds; clippy/fmt clean. Closes Phase 1 of [[tasks/async-only-consolidation]] — one async bus facade, no sync bus path. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 7 +- src/bus/bus.rs | 205 -------- src/bus/event_bus.rs | 10 - src/bus/in_memory_queue.rs | 851 ------------------------------- src/bus/listener.rs | 14 - src/bus/mod.rs | 65 --- src/bus/publisher.rs | 185 ------- src/bus/sender.rs | 13 - src/bus/subscriber.rs | 29 -- src/lib.rs | 14 - src/microsvc/mod.rs | 4 - src/microsvc/service.rs | 363 +------------ src/outbox_worker/mod.rs | 6 - src/outbox_worker/thread.rs | 419 --------------- tests/sagas/handlers/messages.rs | 10 +- 15 files changed, 9 insertions(+), 2186 deletions(-) delete mode 100644 src/bus/bus.rs delete mode 100644 src/bus/event_bus.rs delete mode 100644 src/bus/in_memory_queue.rs delete mode 100644 src/bus/listener.rs delete mode 100644 src/bus/mod.rs delete mode 100644 src/bus/publisher.rs delete mode 100644 src/bus/sender.rs delete mode 100644 src/bus/subscriber.rs delete mode 100644 src/outbox_worker/thread.rs diff --git a/Cargo.toml b/Cargo.toml index 2488aaf25..ae476fa4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,11 +27,10 @@ categories = ["data-structures", "database"] path = "src/lib.rs" [features] -default = ["emitter", "bus"] +default = ["emitter"] emitter = ["dep:event-emitter-rs"] -bus = [] -http = ["bus", "dep:axum", "dep:reqwest", "dep:tokio"] -grpc = ["bus", "dep:tonic", "dep:prost", "dep:tokio"] +http = ["dep:axum", "dep:reqwest", "dep:tokio"] +grpc = ["dep:tonic", "dep:prost", "dep:tokio"] postgres = ["dep:sqlx", "dep:tokio", "sqlx/postgres", "sqlx/runtime-tokio"] sqlite = ["dep:sqlx", "dep:tokio", "sqlx/runtime-tokio", "sqlx/sqlite"] nats = ["dep:async-nats", "dep:futures", "dep:tokio"] diff --git a/src/bus/bus.rs b/src/bus/bus.rs deleted file mode 100644 index fae102fb2..000000000 --- a/src/bus/bus.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! Service Bus - wraps publisher and subscriber for a service. - -use super::in_memory_queue::EventReceiver; -use super::{Event, Listener, PublishError, Publisher, Sender, Subscribable, Subscriber}; - -/// Service bus - wraps publisher and subscriber for a service. -/// -/// The Bus provides a unified interface for both publishing events and -/// subscribing to events. Each service in a distributed system would -/// have its own Bus instance. -/// -/// ## Example -/// -/// ```ignore -/// // Create a bus with separate publisher and subscriber -/// let bus = Bus::new(kafka_publisher, kafka_subscriber); -/// -/// // Or from a unified queue implementation -/// let bus = Bus::from_queue(in_memory_queue); -/// -/// // Publish events -/// bus.publish(Event::with_string_payload("evt-1", "OrderCreated", "{}"))?; -/// -/// // Poll for events -/// if let Some(event) = bus.poll(1000)? { -/// // Process event -/// bus.ack(&event.id)?; -/// } -/// ``` -pub struct Bus { - publisher: P, - subscriber: S, -} - -impl Bus { - /// Create a new bus with the given publisher and subscriber. - pub fn new(publisher: P, subscriber: S) -> Self { - Self { - publisher, - subscriber, - } - } - - /// Publish an event to the bus. - pub fn publish(&self, event: Event) -> Result<(), PublishError> { - self.publisher.publish(event) - } - - /// Publish multiple events to the bus. - pub fn publish_batch(&self, events: Vec) -> Result<(), PublishError> { - self.publisher.publish_batch(events) - } - - /// Poll for the next event, blocking until one is available or timeout. - pub fn poll(&self, timeout_ms: u64) -> Result, PublishError> { - self.subscriber.poll(timeout_ms) - } - - /// Acknowledge that an event has been processed. - pub fn ack(&self, event_id: &str) -> Result<(), PublishError> { - self.subscriber.ack(event_id) - } - - /// Reject an event (will be redelivered or sent to dead letter queue). - pub fn nack(&self, event_id: &str, reason: &str) -> Result<(), PublishError> { - self.subscriber.nack(event_id, reason) - } - - /// Get a reference to the underlying publisher. - pub fn publisher(&self) -> &P { - &self.publisher - } - - /// Get a reference to the underlying subscriber. - pub fn subscriber(&self) -> &S { - &self.subscriber - } -} - -impl Bus { - /// Send an event to a named queue (point-to-point). - /// - /// Unlike `publish` (fan-out), `send` delivers the message to a specific - /// named queue where only one listener consumes each message. - pub fn send(&self, queue: &str, event: Event) -> Result<(), PublishError> { - self.publisher.send(queue, event) - } -} - -impl Bus { - /// Listen for the next event on a named queue (point-to-point). - /// - /// Unlike `poll` (fan-out where each subscriber sees all events), - /// `listen` competes with other listeners — each message is delivered - /// to exactly one listener. - pub fn listen(&self, queue: &str, timeout_ms: u64) -> Result, PublishError> { - self.subscriber.listen(queue, timeout_ms) - } -} - -impl Bus { - /// Subscribe to specific event types, returning a filtered receiver. - /// - /// The returned `EventReceiver` will only deliver events matching the - /// specified types. Other events are skipped. - /// - /// ## Example - /// - /// ```ignore - /// let bus = Bus::from_queue(InMemoryQueue::new()); - /// - /// // Subscribe to specific events - /// let orders = bus.subscribe(&["OrderCreated", "OrderCompleted"]); - /// let payments = bus.subscribe(&["PaymentSucceeded"]); - /// - /// // Each receiver only gets its subscribed event types - /// while let Ok(Some(event)) = orders.recv(100) { - /// // event.event_type is "OrderCreated" or "OrderCompleted" - /// } - /// ``` - pub fn subscribe(&self, event_types: &[&str]) -> EventReceiver { - EventReceiver::new(self.subscriber.new_subscriber(), event_types) - } -} - -// Convenience: when publisher and subscriber are the same type (e.g., InMemoryQueue) -impl Bus { - /// Create a bus from a unified queue that implements both Publisher and Subscriber. - /// - /// This is useful for queue implementations that handle both directions, - /// like InMemoryQueue or some message broker clients. - pub fn from_queue(queue: T) -> Self { - Self { - publisher: queue.clone(), - subscriber: queue, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; - - // Mock publisher for testing - struct MockPublisher { - events: Arc>>, - } - - impl Publisher for MockPublisher { - fn publish(&self, event: Event) -> Result<(), PublishError> { - self.events.lock().unwrap().push(event); - Ok(()) - } - } - - // Mock subscriber for testing - struct MockSubscriber { - events: Arc>>, - position: Arc>, - } - - impl Subscriber for MockSubscriber { - fn poll(&self, _timeout_ms: u64) -> Result, PublishError> { - let events = self.events.lock().unwrap(); - let mut pos = self.position.lock().unwrap(); - if *pos < events.len() { - let event = events[*pos].clone(); - *pos += 1; - Ok(Some(event)) - } else { - Ok(None) - } - } - - fn ack(&self, _event_id: &str) -> Result<(), PublishError> { - Ok(()) - } - - fn nack(&self, _event_id: &str, _reason: &str) -> Result<(), PublishError> { - Ok(()) - } - } - - #[test] - fn bus_publish_and_poll() { - let events = Arc::new(Mutex::new(Vec::new())); - let publisher = MockPublisher { - events: Arc::clone(&events), - }; - let subscriber = MockSubscriber { - events: Arc::clone(&events), - position: Arc::new(Mutex::new(0)), - }; - - let bus = Bus::new(publisher, subscriber); - - bus.publish(Event::with_string_payload("evt-1", "TestEvent", "{}")) - .unwrap(); - - let event = bus.poll(100).unwrap(); - assert!(event.is_some()); - assert_eq!(event.unwrap().event_type, "TestEvent"); - } -} diff --git a/src/bus/event_bus.rs b/src/bus/event_bus.rs deleted file mode 100644 index c28a32f2c..000000000 --- a/src/bus/event_bus.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Combined trait for bidirectional bus communication. - -use super::publisher::Publisher; -use super::subscriber::Subscriber; - -/// Combined trait for bidirectional bus communication. -pub trait EventBus: Publisher + Subscriber {} - -// Blanket implementation -impl EventBus for T {} diff --git a/src/bus/in_memory_queue.rs b/src/bus/in_memory_queue.rs deleted file mode 100644 index 87eaa5592..000000000 --- a/src/bus/in_memory_queue.rs +++ /dev/null @@ -1,851 +0,0 @@ -//! In-memory queue for testing and single-process scenarios. -//! -//! This module provides a thread-safe in-memory queue that implements -//! both `Publisher` and `Subscriber` traits, useful for: -//! - Unit and integration testing without external dependencies -//! - Single-process applications -//! - Development and prototyping - -use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use std::time::{Duration, Instant}; - -use super::{Event, Listener, PublishError, Publisher, Sender, Subscribable, Subscriber}; - -/// Internal data for a named point-to-point queue. -#[derive(Default)] -struct PointToPointQueue { - /// Messages in this queue - messages: Vec, - /// Shared position - all listeners compete for messages - position: usize, -} - -fn lock_poisoned(lock_name: &str) -> PublishError { - PublishError::ConnectionFailed(format!("in-memory queue {lock_name} lock poisoned")) -} - -fn lock_mutex<'a, T>( - mutex: &'a Mutex, - lock_name: &str, -) -> Result, PublishError> { - mutex.lock().map_err(|_| lock_poisoned(lock_name)) -} - -fn read_lock<'a, T>( - lock: &'a RwLock, - lock_name: &str, -) -> Result, PublishError> { - lock.read().map_err(|_| lock_poisoned(lock_name)) -} - -fn write_lock<'a, T>( - lock: &'a RwLock, - lock_name: &str, -) -> Result, PublishError> { - lock.write().map_err(|_| lock_poisoned(lock_name)) -} - -fn recover_mutex(mutex: &Mutex) -> MutexGuard<'_, T> { - match mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } -} - -fn recover_read(lock: &RwLock) -> RwLockReadGuard<'_, T> { - match lock.read() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } -} - -fn recover_write(lock: &RwLock) -> RwLockWriteGuard<'_, T> { - match lock.write() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } -} - -/// In-memory queue for testing and single-process scenarios. -/// -/// Features: -/// - Thread-safe (can be shared across threads via `Clone`) -/// - Supports multiple subscribers via `new_subscriber()` -/// - Events are stored in an append-only log -/// - Each subscriber tracks its own read position -/// -/// ## Example -/// -/// ``` -/// use sourced_rust::bus::{Event, InMemoryQueue, PublishError, Publisher, Subscriber}; -/// -/// fn main() -> Result<(), PublishError> { -/// let queue = InMemoryQueue::new(); -/// -/// queue.publish(Event::with_string_payload("evt-1", "OrderCreated", r#"{"id":"123"}"#))?; -/// -/// let event = queue.poll(100)?.ok_or(PublishError::Timeout)?; -/// assert_eq!(event.event_type, "OrderCreated"); -/// Ok(()) -/// } -/// ``` -/// -/// ## Multiple Subscribers -/// -/// ``` -/// use sourced_rust::bus::{ -/// Event, InMemoryQueue, PublishError, Publisher, Subscribable, Subscriber, -/// }; -/// -/// fn main() -> Result<(), PublishError> { -/// let queue = InMemoryQueue::new(); -/// queue.publish(Event::with_string_payload("evt-1", "Event1", "{}"))?; -/// -/// let sub1 = queue.new_subscriber(); -/// let sub2 = queue.new_subscriber(); -/// -/// assert_eq!(sub1.poll(10)?.ok_or(PublishError::Timeout)?.event_type, "Event1"); -/// assert_eq!(sub2.poll(10)?.ok_or(PublishError::Timeout)?.event_type, "Event1"); -/// Ok(()) -/// } -/// ``` -/// -/// ## Point-to-Point Queues -/// -/// ``` -/// use sourced_rust::bus::{Event, InMemoryQueue, Listener, PublishError, Sender}; -/// -/// fn main() -> Result<(), PublishError> { -/// let queue = InMemoryQueue::new(); -/// queue.send("orders", Event::with_string_payload("evt-1", "ProcessOrder", "{}"))?; -/// -/// let event = queue.listen("orders", 100)?.ok_or(PublishError::Timeout)?; -/// assert_eq!(event.event_type, "ProcessOrder"); -/// Ok(()) -/// } -/// ``` -#[derive(Clone)] -pub struct InMemoryQueue { - /// Shared event log (for pub/sub - fan-out) - log: Arc>>, - /// Per-subscriber read position (for pub/sub) - position: Arc>, - /// Acknowledged event IDs - acked: Arc>>, - /// Named queues for point-to-point messaging (send/listen) - queues: Arc>>, -} - -impl Default for InMemoryQueue { - fn default() -> Self { - Self::new() - } -} - -impl InMemoryQueue { - /// Create a new in-memory queue. - pub fn new() -> Self { - Self { - log: Arc::new(RwLock::new(Vec::new())), - position: Arc::new(Mutex::new(0)), - acked: Arc::new(Mutex::new(Vec::new())), - queues: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Get all events in the log. - pub fn events(&self) -> Vec { - recover_read(&self.log).clone() - } - - /// Get all event types in order. - pub fn event_types(&self) -> Vec { - recover_read(&self.log) - .iter() - .map(|e| e.event_type.clone()) - .collect() - } - - /// Get the total number of events in the log. - pub fn len(&self) -> usize { - recover_read(&self.log).len() - } - - /// Check if the log is empty. - pub fn is_empty(&self) -> bool { - recover_read(&self.log).is_empty() - } - - /// Find an event by type. - pub fn find_by_type(&self, event_type: &str) -> Option { - recover_read(&self.log) - .iter() - .find(|e| e.event_type == event_type) - .cloned() - } - - /// Find all events matching a type. - pub fn find_all_by_type(&self, event_type: &str) -> Vec { - recover_read(&self.log) - .iter() - .filter(|e| e.event_type == event_type) - .cloned() - .collect() - } - - /// Reset the subscriber position to the beginning. - pub fn reset_position(&self) { - *recover_mutex(&self.position) = 0; - } - - /// Get the current subscriber position. - pub fn current_position(&self) -> usize { - *recover_mutex(&self.position) - } - - /// Get acknowledged event IDs. - pub fn acknowledged(&self) -> Vec { - recover_mutex(&self.acked).clone() - } - - /// Clear all events from the log (useful for test cleanup). - pub fn clear(&self) { - self.log.clear_poison(); - self.position.clear_poison(); - self.acked.clear_poison(); - self.queues.clear_poison(); - - recover_write(&self.log).clear(); - *recover_mutex(&self.position) = 0; - recover_mutex(&self.acked).clear(); - recover_write(&self.queues).clear(); - } - - /// Subscribe to specific event types, returning a filtered receiver. - /// - /// The returned `EventReceiver` will only deliver events matching the - /// specified types. Other events are skipped (but still consumed from - /// this subscriber's position). - /// - /// ## Example - /// - /// ``` - /// use sourced_rust::bus::{Event, InMemoryQueue, PublishError, Publisher}; - /// - /// fn main() -> Result<(), PublishError> { - /// let queue = InMemoryQueue::new(); - /// queue.publish(Event::with_string_payload("evt-1", "OrderCreated", "{}"))?; - /// queue.publish(Event::with_string_payload("evt-2", "PaymentFailed", "{}"))?; - /// queue.publish(Event::with_string_payload("evt-3", "OrderCreated", "{}"))?; - /// - /// let receiver = queue.subscribe(&["OrderCreated"]); - /// - /// let event1 = receiver.recv(100)?.ok_or(PublishError::Timeout)?; - /// assert_eq!(event1.event_type, "OrderCreated"); - /// assert_eq!(event1.id, "evt-1"); - /// - /// let event2 = receiver.recv(100)?.ok_or(PublishError::Timeout)?; - /// assert_eq!(event2.event_type, "OrderCreated"); - /// assert_eq!(event2.id, "evt-3"); - /// Ok(()) - /// } - /// ``` - pub fn subscribe(&self, event_types: &[&str]) -> EventReceiver { - EventReceiver::new(self.new_subscriber(), event_types) - } -} - -/// A filtered event receiver that only delivers events of subscribed types. -/// -/// Created via [`InMemoryQueue::subscribe`] or [`Bus::subscribe`]. Each receiver -/// has its own position in the event log and only returns events matching the -/// subscribed types. -pub struct EventReceiver { - subscriber: S, - event_types: HashSet, -} - -impl EventReceiver { - /// Create a new event receiver with the given subscriber and event type filter. - pub fn new(subscriber: S, event_types: &[&str]) -> Self { - Self { - subscriber, - event_types: event_types.iter().map(|s| s.to_string()).collect(), - } - } - - /// Receive the next matching event, blocking until one is available or timeout. - /// - /// Returns `Ok(Some(event))` if a matching event was found, - /// `Ok(None)` if the timeout was reached with no matching events. - pub fn recv(&self, timeout_ms: u64) -> Result, PublishError> { - let deadline = Instant::now() + Duration::from_millis(timeout_ms); - - loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Ok(None); - } - - match self.subscriber.poll(remaining.as_millis() as u64)? { - Some(event) if self.event_types.contains(&event.event_type) => { - return Ok(Some(event)); - } - Some(_) => { - // Skip non-matching event, continue polling - continue; - } - None => { - return Ok(None); - } - } - } - } - - /// Try to receive an event without blocking. - /// - /// Returns immediately with `Ok(None)` if no matching event is available. - pub fn try_recv(&self) -> Result, PublishError> { - self.recv(0) - } - - /// Acknowledge that an event has been processed. - pub fn ack(&self, event_id: &str) -> Result<(), PublishError> { - self.subscriber.ack(event_id) - } - - /// Get the event types this receiver is subscribed to. - pub fn subscribed_types(&self) -> &HashSet { - &self.event_types - } -} - -impl Publisher for InMemoryQueue { - fn publish(&self, event: Event) -> Result<(), PublishError> { - write_lock(&self.log, "event log")?.push(event); - Ok(()) - } - - fn publish_batch(&self, events: Vec) -> Result<(), PublishError> { - let mut log = write_lock(&self.log, "event log")?; - log.extend(events); - Ok(()) - } -} - -impl Subscriber for InMemoryQueue { - fn poll(&self, timeout_ms: u64) -> Result, PublishError> { - let deadline = Instant::now() + Duration::from_millis(timeout_ms); - - loop { - { - let log = read_lock(&self.log, "event log")?; - let mut pos = lock_mutex(&self.position, "subscriber position")?; - - if *pos < log.len() { - let event = log[*pos].clone(); - *pos += 1; - return Ok(Some(event)); - } - } - - if Instant::now() >= deadline { - return Ok(None); - } - - // Small sleep to avoid busy-waiting - std::thread::sleep(Duration::from_millis(1)); - } - } - - fn ack(&self, event_id: &str) -> Result<(), PublishError> { - lock_mutex(&self.acked, "acknowledgement list")?.push(event_id.to_string()); - Ok(()) - } - - fn nack(&self, _event_id: &str, _reason: &str) -> Result<(), PublishError> { - // In-memory queue doesn't support redelivery; events stay in log - Ok(()) - } -} - -impl Subscribable for InMemoryQueue { - fn new_subscriber(&self) -> Self { - Self { - log: Arc::clone(&self.log), - position: Arc::new(Mutex::new(0)), - acked: Arc::new(Mutex::new(Vec::new())), - queues: Arc::clone(&self.queues), - } - } -} - -impl Sender for InMemoryQueue { - fn send(&self, queue: &str, event: Event) -> Result<(), PublishError> { - let mut queues = write_lock(&self.queues, "point-to-point queues")?; - queues - .entry(queue.to_string()) - .or_default() - .messages - .push(event); - Ok(()) - } -} - -impl Listener for InMemoryQueue { - fn listen(&self, queue: &str, timeout_ms: u64) -> Result, PublishError> { - let deadline = Instant::now() + Duration::from_millis(timeout_ms); - - loop { - { - let mut queues = write_lock(&self.queues, "point-to-point queues")?; - if let Some(q) = queues.get_mut(queue) { - if q.position < q.messages.len() { - let event = q.messages[q.position].clone(); - q.position += 1; - return Ok(Some(event)); - } - } - } - - if Instant::now() >= deadline { - return Ok(None); - } - - std::thread::sleep(Duration::from_millis(1)); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn poison_lock(poison: F) - where - F: FnOnce(), - { - assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(poison)).is_err()); - } - - fn assert_lock_poisoned(result: Result, lock_name: &str) { - let err = result.expect_err("poisoned lock should return an error"); - match err { - PublishError::ConnectionFailed(msg) => { - assert!(msg.contains(lock_name), "unexpected lock error: {msg}"); - assert!( - msg.contains("lock poisoned"), - "unexpected lock error: {msg}" - ); - } - other => panic!("unexpected error for poisoned lock: {other:?}"), - } - } - - #[test] - fn publish_and_poll() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload( - "evt-1", - "TestEvent", - r#"{"data": 1}"#, - )) - .unwrap(); - - let event = queue.poll(100).unwrap(); - assert!(event.is_some()); - let event = event.unwrap(); - assert_eq!(event.event_type, "TestEvent"); - assert_eq!(event.payload_str(), Some(r#"{"data": 1}"#)); - } - - #[test] - fn poll_timeout_when_empty() { - let queue = InMemoryQueue::new(); - let event = queue.poll(10).unwrap(); - assert!(event.is_none()); - } - - #[test] - fn multiple_subscribers() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload("evt-1", "Event1", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload("evt-2", "Event2", "{}")) - .unwrap(); - - // Create a second subscriber - let sub2 = queue.new_subscriber(); - - // First subscriber reads both events - assert_eq!(queue.poll(10).unwrap().unwrap().event_type, "Event1"); - assert_eq!(queue.poll(10).unwrap().unwrap().event_type, "Event2"); - - // Second subscriber also reads both events (independent position) - assert_eq!(sub2.poll(10).unwrap().unwrap().event_type, "Event1"); - assert_eq!(sub2.poll(10).unwrap().unwrap().event_type, "Event2"); - } - - #[test] - fn publish_batch() { - let queue = InMemoryQueue::new(); - - let events = vec![ - Event::with_string_payload("evt-1", "Event1", "{}"), - Event::with_string_payload("evt-2", "Event2", "{}"), - Event::with_string_payload("evt-3", "Event3", "{}"), - ]; - - queue.publish_batch(events).unwrap(); - - assert_eq!(queue.len(), 3); - assert_eq!(queue.event_types(), vec!["Event1", "Event2", "Event3"]); - } - - #[test] - fn publish_returns_error_when_log_lock_is_poisoned() { - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.log.write().expect("lock event log"); - panic!("poison event log"); - }); - - assert_lock_poisoned( - queue.publish(Event::with_string_payload("evt-1", "Event1", "{}")), - "event log", - ); - } - - #[test] - fn poll_returns_error_when_position_lock_is_poisoned() { - let queue = InMemoryQueue::new(); - queue - .publish(Event::with_string_payload("evt-1", "Event1", "{}")) - .unwrap(); - poison_lock(|| { - let _guard = queue.position.lock().expect("lock subscriber position"); - panic!("poison subscriber position"); - }); - - assert_lock_poisoned(queue.poll(0), "subscriber position"); - } - - #[test] - fn ack_returns_error_when_ack_lock_is_poisoned() { - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.acked.lock().expect("lock acknowledgement list"); - panic!("poison acknowledgement list"); - }); - - assert_lock_poisoned(queue.ack("evt-1"), "acknowledgement list"); - } - - #[test] - fn send_returns_error_when_queue_lock_is_poisoned() { - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.queues.write().expect("lock point-to-point queues"); - panic!("poison point-to-point queues"); - }); - - assert_lock_poisoned( - queue.send("tasks", Event::with_string_payload("evt-1", "Task", "{}")), - "point-to-point queues", - ); - } - - #[test] - fn accessors_recover_from_poisoned_locks() { - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.log.write().expect("lock event log"); - panic!("poison event log"); - }); - assert!(queue.events().is_empty()); - - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.position.lock().expect("lock subscriber position"); - panic!("poison subscriber position"); - }); - assert_eq!(queue.current_position(), 0); - - let queue = InMemoryQueue::new(); - poison_lock(|| { - let _guard = queue.acked.lock().expect("lock acknowledgement list"); - panic!("poison acknowledgement list"); - }); - assert!(queue.acknowledged().is_empty()); - } - - #[test] - fn find_by_type() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload( - "evt-1", - "OrderCreated", - r#"{"id":"1"}"#, - )) - .unwrap(); - queue - .publish(Event::with_string_payload( - "evt-2", - "PaymentSucceeded", - r#"{"id":"2"}"#, - )) - .unwrap(); - queue - .publish(Event::with_string_payload( - "evt-3", - "OrderCreated", - r#"{"id":"3"}"#, - )) - .unwrap(); - - let found = queue.find_by_type("PaymentSucceeded"); - assert!(found.is_some()); - assert_eq!(found.unwrap().id, "evt-2"); - - let all_orders = queue.find_all_by_type("OrderCreated"); - assert_eq!(all_orders.len(), 2); - } - - #[test] - fn clear_resets_state() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload("evt-1", "Event1", "{}")) - .unwrap(); - queue.poll(10).unwrap(); - queue.ack("evt-1").unwrap(); - queue - .send("tasks", Event::with_string_payload("task-1", "Task", "{}")) - .unwrap(); - - assert_eq!(queue.len(), 1); - assert_eq!(queue.current_position(), 1); - assert_eq!(queue.acknowledged().len(), 1); - assert!(queue.listen("tasks", 0).unwrap().is_some()); - queue - .send("tasks", Event::with_string_payload("task-2", "Task", "{}")) - .unwrap(); - - queue.clear(); - - assert_eq!(queue.len(), 0); - assert_eq!(queue.current_position(), 0); - assert!(queue.acknowledged().is_empty()); - assert!(queue.listen("tasks", 0).unwrap().is_none()); - } - - #[test] - fn clear_resets_lock_poisoning_for_normal_operations() { - let queue = InMemoryQueue::new(); - queue - .publish(Event::with_string_payload("evt-1", "Event1", "{}")) - .unwrap(); - queue.poll(10).unwrap(); - queue.ack("evt-1").unwrap(); - - poison_lock(|| { - let _guard = queue.log.write().expect("lock event log"); - panic!("poison event log"); - }); - poison_lock(|| { - let _guard = queue.position.lock().expect("lock subscriber position"); - panic!("poison subscriber position"); - }); - poison_lock(|| { - let _guard = queue.acked.lock().expect("lock acknowledgement list"); - panic!("poison acknowledgement list"); - }); - poison_lock(|| { - let _guard = queue.queues.write().expect("lock point-to-point queues"); - panic!("poison point-to-point queues"); - }); - - queue.clear(); - - queue - .publish(Event::with_string_payload("evt-2", "Event2", "{}")) - .unwrap(); - assert_eq!(queue.poll(10).unwrap().unwrap().id, "evt-2"); - queue.ack("evt-2").unwrap(); - queue - .send("tasks", Event::with_string_payload("task-1", "Task", "{}")) - .unwrap(); - } - - #[test] - fn subscribe_filters_events() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload("evt-1", "OrderCreated", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload("evt-2", "PaymentFailed", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload( - "evt-3", - "InventoryReserved", - "{}", - )) - .unwrap(); - queue - .publish(Event::with_string_payload("evt-4", "OrderCreated", "{}")) - .unwrap(); - - // Subscribe only to OrderCreated - let receiver = queue.subscribe(&["OrderCreated"]); - - // Should only get OrderCreated events - let event1 = receiver.recv(100).unwrap().unwrap(); - assert_eq!(event1.id, "evt-1"); - assert_eq!(event1.event_type, "OrderCreated"); - - let event2 = receiver.recv(100).unwrap().unwrap(); - assert_eq!(event2.id, "evt-4"); - assert_eq!(event2.event_type, "OrderCreated"); - - // No more matching events - assert!(receiver.recv(10).unwrap().is_none()); - } - - #[test] - fn subscribe_multiple_types() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload("evt-1", "OrderCreated", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload("evt-2", "PaymentFailed", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload("evt-3", "OrderCompleted", "{}")) - .unwrap(); - - // Subscribe to multiple event types - let receiver = queue.subscribe(&["OrderCreated", "OrderCompleted"]); - - let event1 = receiver.recv(100).unwrap().unwrap(); - assert_eq!(event1.id, "evt-1"); - - let event2 = receiver.recv(100).unwrap().unwrap(); - assert_eq!(event2.id, "evt-3"); - - assert!(receiver.recv(10).unwrap().is_none()); - } - - #[test] - fn send_and_listen() { - let queue = InMemoryQueue::new(); - - queue - .send( - "orders", - Event::with_string_payload("evt-1", "ProcessOrder", r#"{"id":"123"}"#), - ) - .unwrap(); - - let event = queue.listen("orders", 100).unwrap(); - assert!(event.is_some()); - let event = event.unwrap(); - assert_eq!(event.event_type, "ProcessOrder"); - assert_eq!(event.id, "evt-1"); - } - - #[test] - fn listen_timeout_when_empty() { - let queue = InMemoryQueue::new(); - let event = queue.listen("orders", 10).unwrap(); - assert!(event.is_none()); - } - - #[test] - fn send_listen_competing_consumers() { - let queue = InMemoryQueue::new(); - - queue - .send("tasks", Event::with_string_payload("evt-1", "Task", "{}")) - .unwrap(); - queue - .send("tasks", Event::with_string_payload("evt-2", "Task", "{}")) - .unwrap(); - - // Two clones share the same queues (competing consumers) - let consumer1 = queue.clone(); - let consumer2 = queue.clone(); - - // Each consumer gets a different message - let e1 = consumer1.listen("tasks", 100).unwrap().unwrap(); - let e2 = consumer2.listen("tasks", 100).unwrap().unwrap(); - assert_eq!(e1.id, "evt-1"); - assert_eq!(e2.id, "evt-2"); - - // No more messages - assert!(queue.listen("tasks", 10).unwrap().is_none()); - } - - #[test] - fn send_listen_separate_queues() { - let queue = InMemoryQueue::new(); - - queue - .send("orders", Event::with_string_payload("evt-1", "Order", "{}")) - .unwrap(); - queue - .send( - "payments", - Event::with_string_payload("evt-2", "Payment", "{}"), - ) - .unwrap(); - - // Each queue is independent - let order = queue.listen("orders", 100).unwrap().unwrap(); - assert_eq!(order.id, "evt-1"); - - let payment = queue.listen("payments", 100).unwrap().unwrap(); - assert_eq!(payment.id, "evt-2"); - - // Queues don't cross-contaminate - assert!(queue.listen("orders", 10).unwrap().is_none()); - assert!(queue.listen("payments", 10).unwrap().is_none()); - } - - #[test] - fn multiple_subscribers_independent() { - let queue = InMemoryQueue::new(); - - queue - .publish(Event::with_string_payload("evt-1", "OrderCreated", "{}")) - .unwrap(); - queue - .publish(Event::with_string_payload( - "evt-2", - "PaymentSucceeded", - "{}", - )) - .unwrap(); - - // Two subscribers with different filters - let orders = queue.subscribe(&["OrderCreated"]); - let payments = queue.subscribe(&["PaymentSucceeded"]); - - // Each gets their own events - assert_eq!(orders.recv(100).unwrap().unwrap().id, "evt-1"); - assert_eq!(payments.recv(100).unwrap().unwrap().id, "evt-2"); - } -} diff --git a/src/bus/listener.rs b/src/bus/listener.rs deleted file mode 100644 index efdea5860..000000000 --- a/src/bus/listener.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Listener trait for point-to-point messaging. - -use super::publisher::{Event, PublishError}; - -/// Trait for listening on a named queue (point-to-point). -/// -/// Unlike `Subscriber` (fan-out where each subscriber sees all events), -/// `Listener` competes with other listeners on the same queue — each -/// message is delivered to exactly one listener. -pub trait Listener: Send + Sync { - /// Listen for the next event on a named queue, blocking until one - /// is available or the timeout expires. - fn listen(&self, queue: &str, timeout_ms: u64) -> Result, PublishError>; -} diff --git a/src/bus/mod.rs b/src/bus/mod.rs deleted file mode 100644 index 424809f25..000000000 --- a/src/bus/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Service Bus - Event publishing abstractions -//! -//! This module provides traits and implementations for publishing events -//! to various message brokers and event buses. -//! -//! ## Architecture -//! -//! ```text -//! ┌─────────────────────────────────────────────────────────────┐ -//! │ Bus (per service) │ -//! │ - Wraps Publisher + Subscriber │ -//! │ - publish() / poll() / ack() │ -//! └─────────────────────────────────────────────────────────────┘ -//! │ -//! ▼ -//! ┌─────────────────────────────────────────────────────────────┐ -//! │ Publisher + Subscriber Traits │ -//! │ Publisher: publish(event) / publish_batch(events) │ -//! │ Subscriber: poll(timeout) / ack(id) / nack(id) │ -//! └─────────────────────────────────────────────────────────────┘ -//! │ │ │ -//! ▼ ▼ ▼ -//! ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ -//! │InMemoryQueue│ │ KafkaQueue │ │ RedisStreamQueue │ -//! │ (included) │ │ (external) │ │ (external) │ -//! └─────────────┘ └─────────────┘ └─────────────────────┘ -//! ``` -//! -//! ## Usage with Outbox Pattern -//! -//! ```ignore -//! // 1. Commit aggregate with outbox message -//! repo.outbox_sync(outbox_msg).commit_sync(&mut order)?; -//! -//! // 2. Worker drains outbox and publishes via bus -//! let bus = Bus::new(kafka_publisher, kafka_subscriber); -//! let outbox = repo.outbox_store(); -//! for msg in outbox.claim(...) { -//! let event = Event::new(msg.id(), &msg.event_type, msg.payload.clone()); -//! bus.publish(event)?; -//! } -//! ``` - -mod bus; -mod event_bus; -mod in_memory_queue; -mod listener; -mod publisher; -mod sender; -mod subscriber; - -pub use bus::Bus; -pub use event_bus::EventBus; -pub use in_memory_queue::{EventReceiver, InMemoryQueue}; -pub use listener::Listener; -pub use publisher::{Event, PublishError, Publisher}; -pub use sender::Sender; -pub use subscriber::{Subscribable, Subscriber}; - -/// Type alias for `Event` when used in a command/message context. -/// -/// Commands and events are both messages — the distinction is in how they're -/// routed: `publish/subscribe` = events (fan-out), `send/listen` = commands -/// (point-to-point). This alias makes command handler signatures read naturally. -pub type Message = Event; diff --git a/src/bus/publisher.rs b/src/bus/publisher.rs deleted file mode 100644 index 4baafd855..000000000 --- a/src/bus/publisher.rs +++ /dev/null @@ -1,185 +0,0 @@ -//! Core publisher traits for the service bus. - -use std::error::Error; -use std::fmt; - -/// An event to be published to the bus. -#[derive(Clone, Debug)] -pub struct Event { - /// Unique identifier for this event - pub id: String, - /// Event type (e.g., "OrderCreated", "PaymentSucceeded") - pub event_type: String, - /// Serialized payload (typically JSON or binary) - pub payload: Vec, - /// Optional metadata (headers, correlation IDs, etc.) - pub metadata: Option>, -} - -impl Event { - /// Create a new event with the given type and payload. - pub fn new(id: impl Into, event_type: impl Into, payload: Vec) -> Self { - Self { - id: id.into(), - event_type: event_type.into(), - payload, - metadata: None, - } - } - - /// Create an event with bitcode-serialized payload. - pub fn encode( - id: impl Into, - event_type: impl Into, - payload: &T, - ) -> Result { - let bytes = bitcode::serialize(payload)?; - Ok(Self::new(id, event_type, bytes)) - } - - /// Decode the payload from bitcode binary format. - pub fn decode(&self) -> Result { - bitcode::deserialize(&self.payload) - } - - /// Create an event with JSON-serialized payload. - pub fn json_encode( - id: impl Into, - event_type: impl Into, - payload: &T, - ) -> Result { - let bytes = serde_json::to_vec(payload)?; - Ok(Self::new(id, event_type, bytes)) - } - - /// Decode the payload from JSON format. - pub fn json_decode(&self) -> Result { - serde_json::from_slice(&self.payload) - } - - /// Create an event with a string payload. - pub fn with_string_payload( - id: impl Into, - event_type: impl Into, - payload: impl Into, - ) -> Self { - Self::new(id, event_type, payload.into().into_bytes()) - } - - /// Add metadata to the event. - pub fn with_metadata(mut self, key: impl Into, value: impl Into) -> Self { - self.metadata - .get_or_insert_with(Vec::new) - .push((key.into(), value.into())); - self - } - - /// Get a metadata value by key. - pub fn meta(&self, key: &str) -> Option<&str> { - self.metadata - .as_ref() - .and_then(|m| m.iter().find(|(k, _)| k == key).map(|(_, v)| v.as_str())) - } - - /// Get the correlation ID, if set. - pub fn correlation_id(&self) -> Option<&str> { - self.meta("correlation_id") - } - - /// Get the causation ID, if set. - pub fn causation_id(&self) -> Option<&str> { - self.meta("causation_id") - } - - /// Get the payload as a string (if valid UTF-8). - pub fn payload_str(&self) -> Option<&str> { - std::str::from_utf8(&self.payload).ok() - } -} - -/// Error type for publish operations. -#[derive(Debug)] -pub enum PublishError { - /// Connection to the bus failed - ConnectionFailed(String), - /// Serialization of the event failed - SerializationFailed(String), - /// The bus rejected the event - Rejected(String), - /// Timeout waiting for acknowledgment - Timeout, - /// Other error - Other(Box), -} - -impl fmt::Display for PublishError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - PublishError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg), - PublishError::SerializationFailed(msg) => write!(f, "Serialization failed: {}", msg), - PublishError::Rejected(msg) => write!(f, "Event rejected: {}", msg), - PublishError::Timeout => write!(f, "Publish timeout"), - PublishError::Other(e) => write!(f, "Publish error: {}", e), - } - } -} - -impl Error for PublishError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - PublishError::Other(e) => Some(e.as_ref()), - _ => None, - } - } -} - -/// Trait for publishing events to a message bus. -/// -/// Implementations might include: -/// - `InMemoryBus` - For testing and single-process scenarios -/// - `KafkaPublisher` - For Apache Kafka -/// - `NatsPublisher` - For NATS -/// - `RabbitMqPublisher` - For RabbitMQ -/// - `CloudEventsPublisher` - For CloudEvents-compatible systems -pub trait Publisher: Send + Sync { - /// Publish a single event to the bus. - fn publish(&self, event: Event) -> Result<(), PublishError>; - - /// Publish multiple events to the bus. - /// - /// Default implementation publishes events sequentially. - /// Implementations may override for batch optimization. - fn publish_batch(&self, events: Vec) -> Result<(), PublishError> { - for event in events { - self.publish(event)?; - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn event_construction() { - let event = Event::new("evt-1", "OrderCreated", b"{}".to_vec()); - assert_eq!(event.id, "evt-1"); - assert_eq!(event.event_type, "OrderCreated"); - assert_eq!(event.payload_str(), Some("{}")); - } - - #[test] - fn event_with_metadata() { - let event = Event::new("evt-1", "OrderCreated", b"{}".to_vec()) - .with_metadata("correlation-id", "abc-123") - .with_metadata("source", "order-service"); - - let meta = event.metadata.unwrap(); - assert_eq!(meta.len(), 2); - assert_eq!( - meta[0], - ("correlation-id".to_string(), "abc-123".to_string()) - ); - } -} diff --git a/src/bus/sender.rs b/src/bus/sender.rs deleted file mode 100644 index 96ad05277..000000000 --- a/src/bus/sender.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Sender trait for point-to-point messaging. - -use super::publisher::{Event, PublishError}; - -/// Trait for sending events to a named queue (point-to-point). -/// -/// Unlike `Publisher` (fan-out to all subscribers), `Sender` delivers -/// messages to a specific named queue where only one listener consumes -/// each message (competing consumers). -pub trait Sender: Send + Sync { - /// Send an event to a named queue. - fn send(&self, queue: &str, event: Event) -> Result<(), PublishError>; -} diff --git a/src/bus/subscriber.rs b/src/bus/subscriber.rs deleted file mode 100644 index cf06203d4..000000000 --- a/src/bus/subscriber.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Core subscriber traits for the service bus. - -use super::publisher::{Event, PublishError}; - -/// Trait for subscribing to events from a message bus. -/// -/// This is a pull-based interface. Implementations may also provide -/// push-based interfaces via callbacks or async streams. -pub trait Subscriber: Send + Sync { - /// Poll for the next event, blocking until one is available or timeout. - fn poll(&self, timeout_ms: u64) -> Result, PublishError>; - - /// Acknowledge that an event has been processed. - fn ack(&self, event_id: &str) -> Result<(), PublishError>; - - /// Reject an event (will be redelivered or sent to dead letter queue). - fn nack(&self, event_id: &str, reason: &str) -> Result<(), PublishError>; -} - -/// Trait for subscribers that can create independent subscriber instances. -/// -/// This enables filtered subscriptions via `Bus::subscribe()`. -pub trait Subscribable: Subscriber + Sized { - /// Create a new independent subscriber sharing the same event source. - /// - /// The new subscriber has its own read position, allowing multiple - /// independent consumers of the same event stream. - fn new_subscriber(&self) -> Self; -} diff --git a/src/lib.rs b/src/lib.rs index c66136d2b..428a0a19a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,6 @@ pub mod aggregate; pub mod entity; pub mod repository; -#[cfg(feature = "bus")] -pub mod bus; mod commit_builder; #[cfg(feature = "emitter")] pub mod emitter; @@ -88,18 +86,6 @@ pub use outbox_worker::{ ProcessOneResult, }; -// Threaded outbox worker (requires bus feature) -#[cfg(feature = "bus")] -pub use outbox_worker::{OutboxWorkerJoinError, OutboxWorkerThread, WorkerStats}; - -// In-memory queue for testing and development (requires bus feature) -#[cfg(feature = "bus")] -pub use bus::InMemoryQueue; - -// Message alias for command contexts (requires bus feature) -#[cfg(feature = "bus")] -pub use bus::Message; - // LocalEmitterPublisher requires the emitter feature #[cfg(feature = "emitter")] pub use outbox_worker::LocalEmitterPublisher; diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index 265120493..d46ee2de8 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -72,10 +72,6 @@ pub use service::{ }; pub use session::Session; -// Bus transports (requires "bus" feature) -#[cfg(feature = "bus")] -pub use service::{listen, subscribe, TransportHandle, TransportJoinError, TransportStats}; - // HTTP transport (requires "http" feature) #[cfg(feature = "http")] mod http; diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index e33fa304e..10b796273 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -20,7 +20,7 @@ //! ``` use std::collections::HashMap; -use std::{error::Error, fmt, sync::Arc}; +use std::sync::Arc; use serde_json::Value; @@ -29,9 +29,6 @@ use super::dependencies::{HasReadModelStore, HasRepo, RepoReadModelDependencies} use super::error::HandlerError; use super::session::Session; -#[cfg(feature = "bus")] -use crate::bus::Event; - type GuardFn = dyn Fn(&Context) -> bool + Send + Sync; type HandlerFn = dyn Fn(&Context) -> Result + Send + Sync; @@ -133,51 +130,7 @@ pub struct Message { pub metadata: Vec<(String, String)>, } -#[cfg(feature = "bus")] -impl From<&Event> for Message { - fn from(event: &Event) -> Self { - Self::from_bus_event(event, MessageKind::Event) - } -} - -#[cfg(feature = "bus")] -impl TryFrom<&Message> for Event { - type Error = HandlerError; - - fn try_from(message: &Message) -> Result { - let id = message - .id - .clone() - .ok_or_else(|| HandlerError::Rejected("message id is required".into()))?; - let metadata = if message.metadata.is_empty() { - None - } else { - Some(message.metadata.clone()) - }; - - Ok(Self { - id, - event_type: message.name.clone(), - payload: message.payload.clone(), - metadata, - }) - } -} - impl Message { - /// Create a transport message from a bus event using an explicit message kind. - #[cfg(feature = "bus")] - pub fn from_bus_event(event: &Event, kind: MessageKind) -> Self { - Self { - id: Some(event.id.clone()), - name: event.event_type.clone(), - kind, - payload: event.payload.clone(), - content_type: "application/json".to_string(), - metadata: event.metadata.clone().unwrap_or_default(), - } - } - /// Create a transport message. pub fn new(name: impl Into, kind: MessageKind, payload: Vec) -> Self { Self { @@ -433,25 +386,6 @@ impl Service { self.invoke(message.clone(), input, session) } - /// Dispatch a bus `Event` as a message. - #[cfg(feature = "bus")] - pub fn dispatch_event(&self, event: &crate::bus::Event) -> Result { - self.dispatch_message(&Message::from(event)) - } - - #[cfg(feature = "bus")] - fn dispatch_listened_event(&self, event: &crate::bus::Event) -> Result { - let kind = if self.handles_message(MessageKind::Command, &event.event_type) { - MessageKind::Command - } else if self.handles_message(MessageKind::Event, &event.event_type) { - MessageKind::Event - } else { - return Err(HandlerError::UnknownCommand(event.event_type.clone())); - }; - - self.dispatch_message(&Message::from_bus_event(event, kind)) - } - fn invoke( &self, message: Message, @@ -556,239 +490,6 @@ impl Service) -> fmt::Result { - write!(f, "microsvc transport thread panicked during shutdown") - } -} - -#[cfg(feature = "bus")] -impl Error for TransportJoinError {} - -/// Handle to a background listener thread. Drop or call `stop()` to shut down. -#[cfg(feature = "bus")] -pub struct TransportHandle { - stop_tx: std::sync::mpsc::Sender<()>, - handle: Option>, -} - -#[cfg(feature = "bus")] -impl TransportHandle { - /// Stop the transport and wait for it to finish. Returns stats. - /// - /// Returns [`TransportJoinError`] if the transport thread panicked before - /// shutdown completed. - pub fn stop(mut self) -> Result { - let _ = self.stop_tx.send(()); - if let Some(handle) = self.handle.take() { - handle.join().map_err(|_| TransportJoinError) - } else { - Ok(TransportStats::default()) - } - } - - /// Signal stop without waiting. - pub fn signal_stop(&self) { - let _ = self.stop_tx.send(()); - } -} - -#[cfg(feature = "bus")] -impl Drop for TransportHandle { - fn drop(&mut self) { - let _ = self.stop_tx.send(()); - } -} - -/// Start listening on a named queue (point-to-point) and dispatching to handlers. -/// -/// Spawns a background thread that polls the queue. Each message is delivered -/// to exactly one listener (competing consumers pattern). -/// -/// The service is wrapped in `Arc` so it can be shared between the transport -/// thread and the caller (for HTTP dispatch, etc.). -/// -/// ## Example -/// -/// ```ignore -/// use std::sync::Arc; -/// use sourced_rust::microsvc; -/// use sourced_rust::bus::{InMemoryQueue, Sender, Event}; -/// -/// let service = Arc::new( -/// sourced_rust::register_handlers!( -/// microsvc::Service::with_repo(repo), -/// command handlers::counter_create, -/// ) -/// ); -/// -/// let queue = InMemoryQueue::new(); -/// let handle = microsvc::listen( -/// service.clone(), -/// "counters", -/// queue.clone(), -/// Duration::from_millis(50), -/// ); -/// -/// // Send commands to the queue -/// queue.send("counters", Event::with_string_payload("cmd-1", "counter.create", r#"{"id":"c1"}"#))?; -/// -/// // HTTP dispatch still works on the same service -/// service.dispatch("counter.create", json!({"id":"c2"}), Session::new())?; -/// -/// let stats = handle.stop()?; -/// ``` -#[cfg(feature = "bus")] -pub fn listen( - service: std::sync::Arc>, - queue_name: &str, - listener: L, - poll_interval: std::time::Duration, -) -> TransportHandle -where - D: Send + Sync + 'static, - L: crate::bus::Listener + 'static, -{ - let queue_name = queue_name.to_string(); - let (stop_tx, stop_rx) = std::sync::mpsc::channel(); - - let handle = std::thread::spawn(move || { - let mut stats = TransportStats::default(); - - loop { - match stop_rx.try_recv() { - Ok(()) | Err(std::sync::mpsc::TryRecvError::Disconnected) => break, - Err(std::sync::mpsc::TryRecvError::Empty) => {} - } - - stats.polls += 1; - - match listener.listen(&queue_name, poll_interval.as_millis() as u64) { - Ok(Some(event)) => match service.dispatch_listened_event(&event) { - Ok(_) => stats.handled += 1, - Err(_) => stats.failed += 1, - }, - Ok(None) => {} - Err(_) => {} - } - } - - stats - }); - - TransportHandle { - stop_tx, - handle: Some(handle), - } -} - -/// Start subscribing to events (pub/sub fan-out) and dispatching to handlers. -/// -/// Spawns a background thread that polls the subscriber. Unlike `listen` -/// (point-to-point), every subscriber sees every event — use this when -/// multiple services need to react to the same events. -/// -/// Successfully handled events are acknowledged. Failed events are nacked. -/// Events with no registered handler are acknowledged and ignored; production -/// transports should use [`Service::subscription_plan`] to avoid delivering -/// unrelated event types to the service. -/// -/// ## Example -/// -/// ```ignore -/// use std::sync::Arc; -/// use sourced_rust::microsvc; -/// use sourced_rust::bus::InMemoryQueue; -/// -/// let service = Arc::new( -/// sourced_rust::register_handlers!( -/// microsvc::Service::new(()), -/// event handlers::on_order_created, -/// ) -/// ); -/// -/// let queue = InMemoryQueue::new(); -/// let handle = microsvc::subscribe( -/// service.clone(), -/// queue.new_subscriber(), -/// Duration::from_millis(50), -/// ); -/// -/// let stats = handle.stop()?; -/// ``` -#[cfg(feature = "bus")] -pub fn subscribe( - service: std::sync::Arc>, - subscriber: S, - poll_interval: std::time::Duration, -) -> TransportHandle -where - D: Send + Sync + 'static, - S: crate::bus::Subscriber + 'static, -{ - let (stop_tx, stop_rx) = std::sync::mpsc::channel(); - - let handle = std::thread::spawn(move || { - let mut stats = TransportStats::default(); - - loop { - match stop_rx.try_recv() { - Ok(()) | Err(std::sync::mpsc::TryRecvError::Disconnected) => break, - Err(std::sync::mpsc::TryRecvError::Empty) => {} - } - - stats.polls += 1; - - match subscriber.poll(poll_interval.as_millis() as u64) { - Ok(Some(event)) if !service.handles_event(&event.event_type) => { - let _ = subscriber.ack(&event.id); - } - Ok(Some(event)) => match service.dispatch_event(&event) { - Ok(_) => { - let _ = subscriber.ack(&event.id); - stats.handled += 1; - } - Err(_) => { - let _ = subscriber.nack(&event.id, "handler error"); - stats.failed += 1; - } - }, - Ok(None) => {} - Err(_) => {} - } - } - - stats - }); - - TransportHandle { - stop_tx, - handle: Some(handle), - } -} - // ============================================================================= // Helpers: convert transport messages to dispatch inputs // ============================================================================= @@ -1130,68 +831,6 @@ mod tests { let result: Result = serde_json::from_str(json); assert!(result.is_err()); } - - #[cfg(feature = "bus")] - #[test] - fn dispatch_event_exposes_raw_payload_without_requiring_json() { - let service = test_service().event("ping").handle(|ctx| { - let payload = std::str::from_utf8(ctx.message().payload()) - .map_err(|err| HandlerError::DecodeFailed(err.to_string()))?; - Ok(json!({ - "event_id": ctx.message().id(), - "payload": payload, - })) - }); - let event = crate::bus::Event::with_string_payload("evt-1", "ping", "not-json"); - let result = service.dispatch_event(&event); - assert_eq!( - result.unwrap(), - json!({ "event_id": "evt-1", "payload": "not-json" }) - ); - } - - #[cfg(feature = "bus")] - #[test] - fn transport_stop_returns_stats_when_thread_exits_cleanly() { - let (stop_tx, stop_rx) = std::sync::mpsc::channel(); - let handle = std::thread::spawn(move || { - let _ = stop_rx.recv(); - TransportStats { - handled: 2, - failed: 1, - polls: 3, - } - }); - let transport = TransportHandle { - stop_tx, - handle: Some(handle), - }; - - let stats = transport.stop().unwrap(); - - assert_eq!(stats.handled, 2); - assert_eq!(stats.failed, 1); - assert_eq!(stats.polls, 3); - } - - #[cfg(feature = "bus")] - #[test] - fn transport_stop_returns_error_when_thread_panics() { - let (stop_tx, _stop_rx) = std::sync::mpsc::channel(); - let handle = std::thread::spawn(|| -> TransportStats { - panic!("transport panic"); - }); - let transport = TransportHandle { - stop_tx, - handle: Some(handle), - }; - - let err = transport - .stop() - .expect_err("transport thread panic should be returned"); - - assert_eq!(err, TransportJoinError); - } } // ============================================================================= diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index f2e0e777e..bfcd2a1e1 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -39,8 +39,6 @@ mod publisher; mod store; -#[cfg(feature = "bus")] -mod thread; mod worker; // Publishers @@ -57,7 +55,3 @@ pub use store::{ // Worker pub use worker::{DrainResult, OutboxWorker, ProcessOneResult}; - -// Threaded worker (requires bus feature) -#[cfg(feature = "bus")] -pub use thread::{OutboxWorkerJoinError, OutboxWorkerThread, WorkerStats}; diff --git a/src/outbox_worker/thread.rs b/src/outbox_worker/thread.rs deleted file mode 100644 index f69bbdde2..000000000 --- a/src/outbox_worker/thread.rs +++ /dev/null @@ -1,419 +0,0 @@ -//! Threaded outbox worker for background message processing. -//! -//! This module provides a background thread that drains the outbox -//! and publishes events to a message bus. - -use std::sync::mpsc::{channel, Sender, TryRecvError}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; -use std::{error::Error, fmt}; - -use crate::bus::{Event, Publisher, Sender as BusSender}; -use crate::{ClaimOutboxMessages, OutboxClaimRef, OutboxStore}; - -const DEFAULT_MAX_ATTEMPTS: u32 = 3; - -/// Statistics from the outbox worker. -#[derive(Debug, Default, Clone)] -pub struct WorkerStats { - pub messages_published: usize, - pub messages_failed: usize, - pub polls: usize, -} - -/// Error returned when an outbox worker thread fails during shutdown. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct OutboxWorkerJoinError; - -impl fmt::Display for OutboxWorkerJoinError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "outbox worker thread panicked during shutdown") - } -} - -impl Error for OutboxWorkerJoinError {} - -fn record_publish_success( - store: &S, - claim: &OutboxClaimRef, - stats: &mut WorkerStats, -) { - match store.complete(claim) { - Ok(()) => { - stats.messages_published += 1; - } - Err(err) => { - eprintln!( - "outbox worker `{}` could not complete `{}`: {err}", - claim.worker_id, claim.message_id - ); - stats.messages_failed += 1; - } - } -} - -fn record_publish_failure( - store: &S, - claim: &OutboxClaimRef, - error: &str, - stats: &mut WorkerStats, -) { - match store.record_failure(claim, error, DEFAULT_MAX_ATTEMPTS) { - Ok(_) => { - stats.messages_failed += 1; - } - Err(err) => { - eprintln!( - "outbox worker `{}` could not record publish failure for `{}`: {err}", - claim.worker_id, claim.message_id - ); - stats.messages_failed += 1; - } - } -} - -/// A background thread that drains outbox messages and publishes to a bus. -/// -/// ## Example -/// -/// ```ignore -/// use sourced_rust::{HashMapRepository, OutboxWorkerThread}; -/// use sourced_rust::bus::Publisher; -/// use std::time::Duration; -/// -/// let repo = HashMapRepository::new(); -/// let publisher = MyPublisher::new(); // implements Publisher -/// -/// // Start the worker -/// let worker = OutboxWorkerThread::spawn( -/// repo.outbox_store(), -/// publisher, -/// Duration::from_millis(50), -/// ); -/// -/// // ... do work ... -/// -/// // Stop the worker and get stats -/// let stats = worker.stop()?; -/// println!("Published {} messages", stats.messages_published); -/// ``` -pub struct OutboxWorkerThread { - stop_tx: Sender<()>, - handle: Option>, -} - -impl OutboxWorkerThread { - /// Spawn a new outbox worker thread. - /// - /// The worker will poll the outbox store for pending outbox messages, - /// publish them to the given publisher, and mark them as complete. - /// - /// The store must be `Send + 'static`. - pub fn spawn(store: S, publisher: P, poll_interval: Duration) -> Self - where - S: OutboxStore + Send + 'static, - P: Publisher + 'static, - { - Self::spawn_with_id(store, publisher, poll_interval, "outbox-worker") - } - - /// Spawn a new outbox worker thread with a custom worker ID. - pub fn spawn_with_id( - store: S, - publisher: P, - poll_interval: Duration, - worker_id: &str, - ) -> Self - where - S: OutboxStore + Send + 'static, - P: Publisher + 'static, - { - let (stop_tx, stop_rx) = channel(); - let worker_id = worker_id.to_string(); - - let handle = thread::spawn(move || { - let mut stats = WorkerStats::default(); - let lease = Duration::from_secs(60); - - loop { - // Check for stop signal - match stop_rx.try_recv() { - Ok(()) | Err(TryRecvError::Disconnected) => break, - Err(TryRecvError::Empty) => {} - } - - stats.polls += 1; - - // Claim and process messages - match store.claim(ClaimOutboxMessages::new(&worker_id, 100, lease)) { - Ok(messages) => { - for msg in messages { - let claim = match OutboxClaimRef::from_message(&msg) { - Ok(claim) => claim, - Err(err) => { - eprintln!( - "outbox worker `{worker_id}` received invalid claim `{}`: {err}", - msg.id() - ); - stats.messages_failed += 1; - continue; - } - }; - let mut event = - Event::new(msg.id(), &msg.event_type, msg.payload.clone()); - for (k, v) in &msg.metadata { - event = event.with_metadata(k, v); - } - - match publisher.publish(event) { - Ok(()) => { - record_publish_success(&store, &claim, &mut stats); - } - Err(err) => { - let error = err.to_string(); - record_publish_failure(&store, &claim, &error, &mut stats); - } - } - } - } - Err(err) => { - eprintln!("outbox worker `{worker_id}` could not claim messages: {err}"); - } - } - - thread::sleep(poll_interval); - } - - stats - }); - - Self { - stop_tx, - handle: Some(handle), - } - } - - /// Spawn a worker that routes messages based on their destination. - /// - /// Messages with a `destination` are sent point-to-point via `Sender::send()`. - /// Messages without a destination are published fan-out via `Publisher::publish()`. - pub fn spawn_routed(store: S, publisher: P, poll_interval: Duration) -> Self - where - S: OutboxStore + Send + 'static, - P: Publisher + BusSender + 'static, - { - Self::spawn_routed_with_id(store, publisher, poll_interval, "outbox-worker") - } - - /// Spawn a routed worker with a custom worker ID. - pub fn spawn_routed_with_id( - store: S, - publisher: P, - poll_interval: Duration, - worker_id: &str, - ) -> Self - where - S: OutboxStore + Send + 'static, - P: Publisher + BusSender + 'static, - { - let (stop_tx, stop_rx) = channel(); - let worker_id = worker_id.to_string(); - - let handle = thread::spawn(move || { - let mut stats = WorkerStats::default(); - let lease = Duration::from_secs(60); - - loop { - match stop_rx.try_recv() { - Ok(()) | Err(TryRecvError::Disconnected) => break, - Err(TryRecvError::Empty) => {} - } - - stats.polls += 1; - - match store.claim(ClaimOutboxMessages::new(&worker_id, 100, lease)) { - Ok(messages) => { - for msg in messages { - let claim = match OutboxClaimRef::from_message(&msg) { - Ok(claim) => claim, - Err(err) => { - eprintln!( - "outbox worker `{worker_id}` received invalid claim `{}`: {err}", - msg.id() - ); - stats.messages_failed += 1; - continue; - } - }; - let mut event = - Event::new(msg.id(), &msg.event_type, msg.payload.clone()); - for (k, v) in &msg.metadata { - event = event.with_metadata(k, v); - } - - let result = if let Some(dest) = &msg.destination { - publisher.send(dest, event) - } else { - publisher.publish(event) - }; - - match result { - Ok(()) => { - record_publish_success(&store, &claim, &mut stats); - } - Err(err) => { - let error = err.to_string(); - record_publish_failure(&store, &claim, &error, &mut stats); - } - } - } - } - Err(err) => { - eprintln!("outbox worker `{worker_id}` could not claim messages: {err}"); - } - } - - thread::sleep(poll_interval); - } - - stats - }); - - Self { - stop_tx, - handle: Some(handle), - } - } - - /// Signal the worker to stop and wait for it to finish. - /// Returns the worker statistics. - /// - /// Returns [`OutboxWorkerJoinError`] if the worker thread panicked before - /// shutdown completed. - pub fn stop(mut self) -> Result { - let _ = self.stop_tx.send(()); - if let Some(handle) = self.handle.take() { - handle.join().map_err(|_| OutboxWorkerJoinError) - } else { - Ok(WorkerStats::default()) - } - } - - /// Signal the worker to stop without waiting. - pub fn signal_stop(&self) { - let _ = self.stop_tx.send(()); - } -} - -impl Drop for OutboxWorkerThread { - fn drop(&mut self) { - let _ = self.stop_tx.send(()); - // Don't join on drop - let the thread finish naturally - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bus::PublishError; - use crate::{CommitBatch, HashMapRepository, OutboxMessage, TransactionalCommit}; - - struct FailingPublisher; - - impl Publisher for FailingPublisher { - fn publish(&self, _event: Event) -> Result<(), PublishError> { - Err(PublishError::Rejected("forced failure".into())) - } - } - - fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { - let id = message.id().to_string(); - let mut batch = CommitBatch::empty(); - batch.outbox_messages.push(message); - repo.commit_batch(batch).unwrap(); - id - } - - fn load_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { - repo.outbox_storage() - .read() - .unwrap() - .get(id) - .unwrap() - .clone() - } - - #[test] - fn stop_returns_stats_when_worker_exits_cleanly() { - let (stop_tx, stop_rx) = channel(); - let handle = thread::spawn(move || { - let _ = stop_rx.recv(); - WorkerStats { - messages_published: 2, - messages_failed: 1, - polls: 3, - } - }); - let worker = OutboxWorkerThread { - stop_tx, - handle: Some(handle), - }; - - let stats = worker.stop().unwrap(); - - assert_eq!(stats.messages_published, 2); - assert_eq!(stats.messages_failed, 1); - assert_eq!(stats.polls, 3); - } - - #[test] - fn stop_returns_error_when_worker_thread_panics() { - let (stop_tx, _stop_rx) = channel(); - let handle = thread::spawn(|| -> WorkerStats { - panic!("worker panic"); - }); - let worker = OutboxWorkerThread { - stop_tx, - handle: Some(handle), - }; - - let err = worker - .stop() - .expect_err("worker thread panic should be returned"); - - assert_eq!(err, OutboxWorkerJoinError); - } - - #[test] - fn worker_thread_fails_message_after_retry_ceiling() { - let repo = HashMapRepository::new(); - let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let id = store_message(&repo, message); - - let worker = OutboxWorkerThread::spawn_with_id( - repo.outbox_store(), - FailingPublisher, - Duration::from_millis(1), - "worker-1", - ); - - for _ in 0..100 { - let stored = load_message(&repo, &id); - if stored.is_failed() { - break; - } - thread::sleep(Duration::from_millis(5)); - } - - let stats = worker.stop().unwrap(); - let stored = load_message(&repo, &id); - - assert!(stored.is_failed()); - assert_eq!(stored.attempts, DEFAULT_MAX_ATTEMPTS); - assert_eq!( - stored.last_error.as_deref(), - Some("Event rejected: forced failure") - ); - assert!(stats.messages_failed >= DEFAULT_MAX_ATTEMPTS as usize); - } -} diff --git a/tests/sagas/handlers/messages.rs b/tests/sagas/handlers/messages.rs index 3e7966be8..a97b862ad 100644 --- a/tests/sagas/handlers/messages.rs +++ b/tests/sagas/handlers/messages.rs @@ -1,7 +1,7 @@ //! Shared message types for inter-service communication. //! -//! Each message is serialized as JSON via [`json_outbox_to`] so that -//! `microsvc::dispatch_event` can deserialize it on the receiving end. +//! Each message is serialized as JSON via [`json_outbox_to`] so the receiving +//! service can decode it from the bus message payload (`ctx.input`). use serde::{Deserialize, Serialize}; use sourced_rust::microsvc::HandlerError; @@ -11,9 +11,9 @@ use crate::order::OrderItem; /// Create a JSON-serialized outbox message routed to a destination queue. /// -/// `OutboxMessage::encode_to` uses bitcode internally, but the microsvc -/// transport layer (`dispatch_event`) expects JSON. This helper bridges the -/// gap by using `serde_json::to_vec` + `OutboxMessage::create_to`. +/// `OutboxMessage::encode_to` uses bitcode internally, but the receiving +/// handlers decode the bus message payload as JSON (`ctx.input`). This helper +/// bridges the gap with `serde_json::to_vec` + `OutboxMessage::create_to`. pub fn json_outbox_to( id: &str, event_type: &str, From 27f58eaffcb7d7d72fd47c298097a473822a92c6 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 19:19:29 -0500 Subject: [PATCH 35/39] =?UTF-8?q?feat(microsvc)!:=20async=20handler=20mode?= =?UTF-8?q?l=20(core)=20=E2=80=94=20handlers=20become=20async=20fn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts the microsvc handler model from sync to async, the foundation for dropping the sync repository API so all backends are async-only (the sync/async mix was the source of subtle bugs). Core (lib green; integration test crates migrated in follow-up commits): - HandlerFn is now `dyn for<'a> Fn(&'a Context<'a, D>) -> Pin> + Send + 'a>>`, with an `AsyncHandler<'a,D>` HRTB helper trait so `async fn handle(ctx: &Context)` registers directly. Guards stay synchronous. - Service::dispatch / dispatch_message / dispatch_request / invoke are async. - dependencies.rs: HasRepo/HasReadModelStore now resolve via the ASYNC repo + read-model traits (+ HasRepo for AsyncAggregateRepository / AsyncSnapshotAggregateRepository). - run_source + the http/grpc/knative transports await dispatch. - src unit tests converted (async-closure handlers + awaited dispatch). Handler authors write `async fn handle`; closures need an explicit ctx type annotation and must extract owned values before the `async move` (the future cannot borrow ctx across the await — an HRTB-closure limitation). cargo build (default + --features http,grpc) green; 238 lib tests pass. NOTE: tests/ integration crates still use the sync handler API and are migrated in the following commits (all-or-nothing handler switch). Part of [[tasks/async-only-consolidation]] (Phase 3). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/dependencies.rs | 17 +- src/microsvc/grpc.rs | 2 +- src/microsvc/http.rs | 2 +- src/microsvc/service.rs | 335 +++++++++++++++--------- src/microsvc/transport/in_memory_bus.rs | 43 +-- src/microsvc/transport/knative.rs | 2 +- src/microsvc/transport/outbox_source.rs | 23 +- src/microsvc/transport/runner.rs | 35 ++- 8 files changed, 294 insertions(+), 165 deletions(-) diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index 383f618fb..8168ed81f 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -1,9 +1,10 @@ //! Typed dependency wrappers for microsvc handlers. -use crate::aggregate::AggregateRepository; -use crate::read_model::{ReadModelWritePlanStore, RelationalReadModelQueryStore}; -use crate::repository::Repository; -use crate::snapshot::SnapshotAggregateRepository; +use crate::aggregate::AsyncAggregateRepository; +use crate::repository::{ + AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, +}; +use crate::snapshot::AsyncSnapshotAggregateRepository; /// Dependency capability for services that expose an aggregate repository. pub trait HasRepo { @@ -21,7 +22,7 @@ pub trait HasReadModelStore { impl HasRepo for R where - R: Repository, + R: AsyncRepository, { type Repo = R; @@ -30,7 +31,7 @@ where } } -impl HasRepo for AggregateRepository { +impl HasRepo for AsyncAggregateRepository { type Repo = Self; fn repo(&self) -> &Self::Repo { @@ -38,7 +39,7 @@ impl HasRepo for AggregateRepository { } } -impl HasRepo for SnapshotAggregateRepository { +impl HasRepo for AsyncSnapshotAggregateRepository { type Repo = Self; fn repo(&self) -> &Self::Repo { @@ -48,7 +49,7 @@ impl HasRepo for SnapshotAggregateRepository { impl HasReadModelStore for S where - S: ReadModelWritePlanStore + RelationalReadModelQueryStore, + S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, { type ReadModelStore = S; diff --git a/src/microsvc/grpc.rs b/src/microsvc/grpc.rs index 7e45db362..5cc6c1049 100644 --- a/src/microsvc/grpc.rs +++ b/src/microsvc/grpc.rs @@ -163,7 +163,7 @@ impl CommandService for GrpcHandler { // Build session: start with metadata headers, then overlay payload values let session = build_session(&metadata, req.session_variables); - match self.service.dispatch(&req.command, input, session) { + match self.service.dispatch(&req.command, input, session).await { Ok(value) => Ok(Response::new(GrpcResponse { status: 200, body: value.to_string(), diff --git a/src/microsvc/http.rs b/src/microsvc/http.rs index 111a01bad..1f32e23d1 100644 --- a/src/microsvc/http.rs +++ b/src/microsvc/http.rs @@ -73,7 +73,7 @@ async fn command_handler( Json(input): Json, ) -> impl IntoResponse { let session = session_from_headers(&headers); - match service.dispatch(&command, input, session) { + match service.dispatch(&command, input, session).await { Ok(value) => (StatusCode::OK, Json(value)).into_response(), Err(err) => { let status = status_for_error(&err); diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index 10b796273..dcf300d23 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -20,6 +20,8 @@ //! ``` use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use serde_json::Value; @@ -30,7 +32,37 @@ use super::error::HandlerError; use super::session::Session; type GuardFn = dyn Fn(&Context) -> bool + Send + Sync; -type HandlerFn = dyn Fn(&Context) -> Result + Send + Sync; +type HandlerFuture<'a> = Pin> + Send + 'a>>; +type HandlerFn = dyn for<'a> Fn(&'a Context<'a, D>) -> HandlerFuture<'a> + Send + Sync; + +/// Lets an `async fn handle(ctx: &Context) -> Result` +/// register directly as a handler. The higher-ranked bound ties the returned +/// future's lifetime to the borrowed [`Context`], which a plain generic future +/// parameter cannot express. +pub trait AsyncHandler<'a, D: 'a>: Send + Sync { + /// The future returned by the handler for a context borrowed for `'a`. + type Future: Future> + Send + 'a; + fn call(&self, ctx: &'a Context<'a, D>) -> Self::Future; +} + +impl<'a, D, F, Fut> AsyncHandler<'a, D> for F +where + D: 'a, + F: Fn(&'a Context<'a, D>) -> Fut + Send + Sync, + Fut: Future> + Send + 'a, +{ + type Future = Fut; + fn call(&self, ctx: &'a Context<'a, D>) -> Fut { + self(ctx) + } +} + +fn boxed_handler(handler: F) -> Arc> +where + F: for<'a> AsyncHandler<'a, D> + 'static, +{ + Arc::new(move |ctx| Box::pin(handler.call(ctx)) as HandlerFuture<'_>) +} /// The kind of message a handler consumes. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Deserialize, serde::Serialize)] @@ -223,23 +255,23 @@ pub struct HandlerBuilder { } impl HandlerBuilder { - /// Register a handler without a guard. + /// Register an async handler without a guard. pub fn handle(self, handler: F) -> Service where - F: Fn(&Context) -> Result + Send + Sync + 'static, + F: for<'a> AsyncHandler<'a, D> + 'static, { self.service - .register_handler(self.spec, None, Arc::new(handler)) + .register_handler(self.spec, None, boxed_handler(handler)) } - /// Register a handler with a guard. + /// Register an async handler with a (synchronous) guard. pub fn guarded(self, guard: G, handler: F) -> Service where G: Fn(&Context) -> bool + Send + Sync + 'static, - F: Fn(&Context) -> Result + Send + Sync + 'static, + F: for<'a> AsyncHandler<'a, D> + 'static, { self.service - .register_handler(self.spec, Some(Arc::new(guard)), Arc::new(handler)) + .register_handler(self.spec, Some(Arc::new(guard)), boxed_handler(handler)) } } @@ -327,7 +359,7 @@ impl Service { /// /// Builds a `Context` from the input and session, looks up the handler, /// runs the guard (if any), then calls the handler. - pub fn dispatch( + pub async fn dispatch( &self, command: &str, input: Value, @@ -354,13 +386,16 @@ impl Service { metadata, }; - self.invoke(message, input, session) + self.invoke(message, input, session).await } /// Dispatch a `CommandRequest`, returning a `CommandResponse`. - pub fn dispatch_request(&self, request: &CommandRequest) -> CommandResponse { + pub async fn dispatch_request(&self, request: &CommandRequest) -> CommandResponse { let session = Session::from_map(request.session_variables.clone()); - match self.dispatch(&request.command, request.input.clone(), session) { + match self + .dispatch(&request.command, request.input.clone(), session) + .await + { Ok(value) => CommandResponse { status: 200, body: value, @@ -373,7 +408,7 @@ impl Service { } /// Dispatch a transport message. - pub fn dispatch_message(&self, message: &Message) -> Result { + pub async fn dispatch_message(&self, message: &Message) -> Result { if !self.handles_message(message.kind, &message.name) { return Err(HandlerError::UnknownCommand(message.name.clone())); } @@ -383,30 +418,35 @@ impl Service { Err(_) => Value::Null, }; let session = message_to_session(message); - self.invoke(message.clone(), input, session) + self.invoke(message.clone(), input, session).await } - fn invoke( + async fn invoke( &self, message: Message, input: Value, session: Session, ) -> Result { - let handler = self - .handlers - .get(&handler_key(message.kind, &message.name)) - .ok_or_else(|| HandlerError::UnknownCommand(message.name.clone()))?; + // Clone the handler/guard Arcs so the handler map is not borrowed across + // the (awaited) handler future. + let (guard, handle) = { + let handler = self + .handlers + .get(&handler_key(message.kind, &message.name)) + .ok_or_else(|| HandlerError::UnknownCommand(message.name.clone()))?; + (handler.guard.clone(), handler.handle.clone()) + }; let name = message.name.clone(); let ctx = Context::new(message, input, session, &self.dependencies); - // Run guard if present - if let Some(guard) = &handler.guard { + // Run guard (synchronous) if present. + if let Some(guard) = &guard { if !guard(&ctx) { return Err(HandlerError::GuardRejected(name)); } } - (handler.handle.as_ref())(&ctx) + handle(&ctx).await } /// List registered command names. @@ -539,43 +579,53 @@ mod tests { Service::new(()) } - #[test] - fn dispatch_returns_handler_result() { + #[tokio::test] + async fn dispatch_returns_handler_result() { let service = test_service() .command("ping") - .handle(|_ctx| Ok(json!({ "pong": true }))); - let result = service.dispatch("ping", json!({}), Session::new()).unwrap(); + .handle(|_ctx: &Context<()>| async move { Ok(json!({ "pong": true })) }); + let result = service + .dispatch("ping", json!({}), Session::new()) + .await + .unwrap(); assert_eq!(result, json!({ "pong": true })); } - #[test] - fn unknown_command() { - let service = test_service().command("ping").handle(|_ctx| Ok(json!({}))); - let result = service.dispatch("unknown", json!({}), Session::new()); + #[tokio::test] + async fn unknown_command() { + let service = test_service() + .command("ping") + .handle(|_ctx: &Context<()>| async move { Ok(json!({})) }); + let result = service.dispatch("unknown", json!({}), Session::new()).await; assert!(matches!(result, Err(HandlerError::UnknownCommand(ref s)) if s == "unknown")); } - #[test] - fn handler_error_propagates() { + #[tokio::test] + async fn handler_error_propagates() { let service = test_service() .command("fail") - .handle(|_ctx| Err(HandlerError::Rejected("nope".into()))); - let result = service.dispatch("fail", json!({}), Session::new()); + .handle(|_ctx: &Context<()>| async move { Err(HandlerError::Rejected("nope".into())) }); + let result = service.dispatch("fail", json!({}), Session::new()).await; assert!(matches!(result, Err(HandlerError::Rejected(ref s)) if s == "nope")); } - #[test] - fn decode_error_from_bad_payload() { + #[tokio::test] + async fn decode_error_from_bad_payload() { #[derive(serde::Deserialize)] struct Input { _name: String, } - let service = test_service().command("typed").handle(|ctx| { - let _input = ctx.input::()?; - Ok(json!({})) + let service = test_service().command("typed").handle(|ctx: &Context<()>| { + let input = ctx.input::(); + async move { + let _input = input?; + Ok(json!({})) + } }); - let result = service.dispatch("typed", json!({ "wrong": 1 }), Session::new()); + let result = service + .dispatch("typed", json!({ "wrong": 1 }), Session::new()) + .await; assert!(matches!(result, Err(HandlerError::DecodeFailed(_)))); } @@ -583,9 +633,9 @@ mod tests { fn command_names_list() { let service = test_service() .command("a") - .handle(|_| Ok(json!({}))) + .handle(|_: &Context<()>| async move { Ok(json!({})) }) .command("b") - .handle(|_| Ok(json!({}))); + .handle(|_: &Context<()>| async move { Ok(json!({})) }); let mut cmds = service.command_names(); cmds.sort(); assert_eq!(cmds, vec!["a", "b"]); @@ -597,9 +647,9 @@ mod tests { let service = test_service() .command("checkout.start") - .handle(|_| Ok(json!({}))) + .handle(|_: &Context<()>| async move { Ok(json!({})) }) .events(EVENTS) - .guarded(|_| true, |_| Ok(json!({}))); + .guarded(|_| true, |_: &Context<()>| async move { Ok(json!({})) }); assert_eq!( service.subscription_plan(), @@ -616,9 +666,9 @@ mod tests { let service = test_service() .event("checkout.started") - .handle(|_| Ok(json!({}))) + .handle(|_: &Context<()>| async move { Ok(json!({})) }) .events(EVENTS) - .handle(|_| Ok(json!({}))); + .handle(|_: &Context<()>| async move { Ok(json!({})) }); let mut events = service.event_names(); events.sort(); @@ -628,20 +678,27 @@ mod tests { ); } - #[test] - fn command_and_event_handlers_can_share_a_name() { + #[tokio::test] + async fn command_and_event_handlers_can_share_a_name() { let service = test_service() .command("shared") - .handle(|ctx| Ok(json!({ "kind": format!("{:?}", ctx.message().kind) }))) + .handle(|ctx: &Context<()>| { + let kind = format!("{:?}", ctx.message().kind); + async move { Ok(json!({ "kind": kind })) } + }) .event("shared") - .handle(|ctx| Ok(json!({ "event_id": ctx.message().id() }))); + .handle(|ctx: &Context<()>| { + let event_id = ctx.message().id().map(|s| s.to_string()); + async move { Ok(json!({ "event_id": event_id })) } + }); let event_message = Message::new("shared", MessageKind::Event, br#"{}"#.to_vec()).with_id("evt-1"); let command_result = service .dispatch("shared", json!({}), Session::new()) + .await .unwrap(); - let event_result = service.dispatch_message(&event_message).unwrap(); + let event_result = service.dispatch_message(&event_message).await.unwrap(); assert_eq!(command_result, json!({ "kind": "Command" })); assert_eq!(event_result, json!({ "event_id": "evt-1" })); @@ -649,19 +706,29 @@ mod tests { assert!(service.handles_message(MessageKind::Event, "shared")); } - #[test] - fn dispatch_message_delivers_payload_json_by_default() { - let service = test_service().event("checkout.started").handle(|ctx| { - if !ctx.has_fields(&["checkout_id"]) { - return Err(HandlerError::Rejected("missing checkout_id".into())); - } - - Ok(json!({ - "event_id": ctx.message().id(), - "checkout_id": ctx.raw_input()["checkout_id"].as_str().unwrap(), - "user_id": ctx.user_id()?, - })) - }); + #[tokio::test] + async fn dispatch_message_delivers_payload_json_by_default() { + let service = test_service() + .event("checkout.started") + .handle(|ctx: &Context<()>| { + let has_checkout_id = ctx.has_fields(&["checkout_id"]); + let event_id = ctx.message().id().map(|s| s.to_string()); + let checkout_id = ctx.raw_input()["checkout_id"] + .as_str() + .map(|s| s.to_string()); + let user_id = ctx.user_id().map(|s| s.to_string()); + async move { + if !has_checkout_id { + return Err(HandlerError::Rejected("missing checkout_id".into())); + } + + Ok(json!({ + "event_id": event_id, + "checkout_id": checkout_id.unwrap(), + "user_id": user_id?, + })) + } + }); let message = Message { id: Some("evt-1".to_string()), name: "checkout.started".to_string(), @@ -671,7 +738,7 @@ mod tests { metadata: vec![("X-Hasura-User-Id".to_string(), "user-1".to_string())], }; - let result = service.dispatch_message(&message).unwrap(); + let result = service.dispatch_message(&message).await.unwrap(); assert_eq!( result, @@ -679,19 +746,25 @@ mod tests { ); } - #[test] - fn dispatch_message_always_exposes_message_metadata() { + #[tokio::test] + async fn dispatch_message_always_exposes_message_metadata() { let service = test_service().event("seat.reserved").guarded( |ctx| ctx.message().id().is_some(), - |ctx| { - let input: Value = ctx.input()?; + |ctx: &Context<()>| { + let input: Result = ctx.input(); let message = ctx.message(); - Ok(json!({ - "event_id": message.id(), - "name": message.name(), - "correlation_id": message.correlation_id(), - "seat_id": input["seat_id"].as_str().unwrap(), - })) + let event_id = message.id().map(|s| s.to_string()); + let name = message.name().to_string(); + let correlation_id = message.correlation_id().map(|s| s.to_string()); + async move { + let input = input?; + Ok(json!({ + "event_id": event_id, + "name": name, + "correlation_id": correlation_id, + "seat_id": input["seat_id"].as_str().unwrap(), + })) + } }, ); let message = Message { @@ -703,7 +776,7 @@ mod tests { metadata: vec![("Correlation_ID".to_string(), "checkout-1".to_string())], }; - let result = service.dispatch_message(&message).unwrap(); + let result = service.dispatch_message(&message).await.unwrap(); assert_eq!( result, @@ -716,103 +789,125 @@ mod tests { ); } - #[test] - fn guard_passes() { + #[tokio::test] + async fn guard_passes() { let service = test_service().command("greet").guarded( |ctx| ctx.has_fields(&["name"]), - |ctx| { - let name = ctx.raw_input()["name"].as_str().unwrap(); - Ok(json!({ "hello": name })) + |ctx: &Context<()>| { + let name = ctx.raw_input()["name"].as_str().map(|s| s.to_string()); + async move { Ok(json!({ "hello": name.unwrap() })) } }, ); let result = service .dispatch("greet", json!({ "name": "Pat" }), Session::new()) + .await .unwrap(); assert_eq!(result, json!({ "hello": "Pat" })); } - #[test] - fn guard_rejects() { + #[tokio::test] + async fn guard_rejects() { let service = test_service().command("greet").guarded( |ctx| ctx.has_fields(&["name"]), - |_ctx| panic!("handler should not run"), + |_ctx: &Context<()>| async move { + panic!("handler should not run"); + #[allow(unreachable_code)] + Ok(json!({})) + }, ); - let result = service.dispatch("greet", json!({ "wrong": 1 }), Session::new()); + let result = service + .dispatch("greet", json!({ "wrong": 1 }), Session::new()) + .await; assert!(matches!(result, Err(HandlerError::GuardRejected(ref s)) if s == "greet")); } - #[test] - fn guard_checks_session() { + #[tokio::test] + async fn guard_checks_session() { let service = test_service().command("admin").guarded( |ctx| ctx.role() == Some("admin"), - |_ctx| Ok(json!({ "ok": true })), + |_ctx: &Context<()>| async move { Ok(json!({ "ok": true })) }, ); // No role assert!(service .dispatch("admin", json!({}), Session::new()) + .await .is_err()); // Admin role let mut session = Session::new(); session.set("x-hasura-role", "admin"); - assert!(service.dispatch("admin", json!({}), session).is_ok()); + assert!(service.dispatch("admin", json!({}), session).await.is_ok()); } - #[test] - fn dispatch_request_success() { + #[tokio::test] + async fn dispatch_request_success() { let service = test_service() .command("ping") - .handle(|_ctx| Ok(json!({ "pong": true }))); + .handle(|_ctx: &Context<()>| async move { Ok(json!({ "pong": true })) }); let request = CommandRequest { command: "ping".to_string(), input: json!({}), session_variables: HashMap::new(), }; - let response = service.dispatch_request(&request); + let response = service.dispatch_request(&request).await; assert_eq!(response.status, 200); assert_eq!(response.body, json!({ "pong": true })); } - #[test] - fn dispatch_request_error_codes() { + #[tokio::test] + async fn dispatch_request_error_codes() { let service = test_service() .command("reject") - .handle(|_| Err(HandlerError::Rejected("no".into()))) + .handle(|_: &Context<()>| async move { Err(HandlerError::Rejected("no".into())) }) .command("unauth") - .handle(|ctx| { - let _ = ctx.user_id()?; - Ok(json!({})) + .handle(|ctx: &Context<()>| { + let user_id = ctx.user_id().map(|s| s.to_string()); + async move { + let _ = user_id?; + Ok(json!({})) + } }); - let resp = service.dispatch_request(&CommandRequest { - command: "unknown".to_string(), - input: json!({}), - session_variables: HashMap::new(), - }); + let resp = service + .dispatch_request(&CommandRequest { + command: "unknown".to_string(), + input: json!({}), + session_variables: HashMap::new(), + }) + .await; assert_eq!(resp.status, 404); - let resp = service.dispatch_request(&CommandRequest { - command: "reject".to_string(), - input: json!({}), - session_variables: HashMap::new(), - }); + let resp = service + .dispatch_request(&CommandRequest { + command: "reject".to_string(), + input: json!({}), + session_variables: HashMap::new(), + }) + .await; assert_eq!(resp.status, 422); - let resp = service.dispatch_request(&CommandRequest { - command: "unauth".to_string(), - input: json!({}), - session_variables: HashMap::new(), - }); + let resp = service + .dispatch_request(&CommandRequest { + command: "unauth".to_string(), + input: json!({}), + session_variables: HashMap::new(), + }) + .await; assert_eq!(resp.status, 401); } - #[test] - fn dispatch_request_passes_session() { - let service = test_service().command("whoami").handle(|ctx| { - let user_id = ctx.user_id()?; - Ok(json!({ "user_id": user_id })) - }); + #[tokio::test] + async fn dispatch_request_passes_session() { + let service = test_service() + .command("whoami") + .handle(|ctx: &Context<()>| { + let user_id = ctx.user_id().map(|s| s.to_string()); + async move { + let user_id = user_id?; + Ok(json!({ "user_id": user_id })) + } + }); let mut vars = HashMap::new(); vars.insert("x-hasura-user-id".to_string(), "user-99".to_string()); let request = CommandRequest { @@ -820,7 +915,7 @@ mod tests { input: json!({}), session_variables: vars, }; - let response = service.dispatch_request(&request); + let response = service.dispatch_request(&request).await; assert_eq!(response.status, 200); assert_eq!(response.body, json!({ "user_id": "user-99" })); } diff --git a/src/microsvc/transport/in_memory_bus.rs b/src/microsvc/transport/in_memory_bus.rs index 01929a79e..464f18764 100644 --- a/src/microsvc/transport/in_memory_bus.rs +++ b/src/microsvc/transport/in_memory_bus.rs @@ -212,19 +212,31 @@ mod tests { } fn command_service(rec: Arc>>) -> Arc> { - Arc::new(Service::new(()).command("work").handle(move |ctx| { - rec.lock().unwrap().push(ctx.message().name().to_string()); - Ok(json!({})) - })) + Arc::new(Service::new(()).command("work").handle( + move |ctx: &crate::microsvc::Context<()>| { + let rec = rec.clone(); + let name = ctx.message().name().to_string(); + async move { + rec.lock().unwrap().push(name); + Ok(json!({})) + } + }, + )) } fn event_service(rec: Arc>>) -> Arc> { - Arc::new(Service::new(()).event("evt").handle(move |ctx| { - rec.lock() - .unwrap() - .push(ctx.message().id().unwrap_or("?").to_string()); - Ok(json!({})) - })) + Arc::new( + Service::new(()) + .event("evt") + .handle(move |ctx: &crate::microsvc::Context<()>| { + let rec = rec.clone(); + let id = ctx.message().id().unwrap_or("?").to_string(); + async move { + rec.lock().unwrap().push(id); + Ok(json!({})) + } + }), + ) } #[test] @@ -318,11 +330,12 @@ mod tests { fn handler_error_does_not_panic_the_loop() { let bus = InMemoryBus::new(); block_on(bus.send("work", b"{}".to_vec())).unwrap(); - let service: Arc> = Arc::new( - Service::new(()) - .command("work") - .handle(|_| Err(HandlerError::Rejected("no".into()))), - ); + let service: Arc> = + Arc::new(Service::new(()).command("work").handle( + |_: &crate::microsvc::Context<()>| async move { + Err(HandlerError::Rejected("no".into())) + }, + )); // Default failure policy dead-letters the permanent failure; in-memory // dead_letter is a no-op nack, so the run completes cleanly. block_on(bus.listen(service, RunOptions::idempotent())).unwrap(); diff --git a/src/microsvc/transport/knative.rs b/src/microsvc/transport/knative.rs index d06a2473b..ca66e4e86 100644 --- a/src/microsvc/transport/knative.rs +++ b/src/microsvc/transport/knative.rs @@ -58,7 +58,7 @@ async fn ingress_handler( Err(reason) => return (StatusCode::BAD_REQUEST, reason).into_response(), }; - match service.dispatch_message(&message) { + match service.dispatch_message(&message).await { Ok(value) => (StatusCode::OK, Json(value)).into_response(), Err(err) => { // Map our retryable/permanent classification onto HTTP so Knative's diff --git a/src/microsvc/transport/outbox_source.rs b/src/microsvc/transport/outbox_source.rs index 1cb4a40eb..7ed487a22 100644 --- a/src/microsvc/transport/outbox_source.rs +++ b/src/microsvc/transport/outbox_source.rs @@ -305,12 +305,16 @@ mod tests { let handled = Arc::new(std::sync::Mutex::new(Vec::::new())); let h = handled.clone(); - let service = Arc::new(Service::new(()).event("evt").handle(move |ctx| { - h.lock() - .unwrap() - .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) - })); + let service = Arc::new(Service::new(()).event("evt").handle( + move |ctx: &crate::microsvc::Context<()>| { + let h = h.clone(); + let id = ctx.message().id().unwrap_or_default().to_string(); + async move { + h.lock().unwrap().push(id); + Ok(json!({})) + } + }, + )); block_on(run_source(service, source(&repo), RunOptions::idempotent())).unwrap(); @@ -327,8 +331,11 @@ mod tests { store_row(&repo, "m1", "unrelated"); // Service handles a different event; the unrelated row is acked-ignored, // i.e. completed, so it does not loop forever. - let service: Arc> = - Arc::new(Service::new(()).event("evt").handle(|_| Ok(json!({})))); + let service: Arc> = Arc::new( + Service::new(()) + .event("evt") + .handle(|_: &crate::microsvc::Context<()>| async move { Ok(json!({})) }), + ); block_on(run_source(service, source(&repo), RunOptions::idempotent())).unwrap(); assert_eq!(status(&repo, "m1"), Some(OutboxMessageStatus::Published)); } diff --git a/src/microsvc/transport/runner.rs b/src/microsvc/transport/runner.rs index 55612385b..be6ae6c54 100644 --- a/src/microsvc/transport/runner.rs +++ b/src/microsvc/transport/runner.rs @@ -60,7 +60,7 @@ where received.ack().await?; continue; } - match dispatch(&service, &options, received.message()) { + match dispatch(&service, &options, received.message()).await { Ok(()) => received.ack().await?, Err(error) => match options.failure_policy.resolve(&error) { FailureAction::Nack => received.nack(&error.to_string()).await?, @@ -85,7 +85,7 @@ where /// Enforces the inbox stable-id contract first (idempotent mode yields no key /// and skips it), then dispatches. A failed stable-id check is a permanent /// failure — redelivery cannot supply a missing or malformed id. -fn dispatch( +async fn dispatch( service: &Service, options: &RunOptions, message: &Message, @@ -98,6 +98,7 @@ where .map_err(|err| TransportError::permanent(err.to_string()).with_source(err))?; service .dispatch_message(message) + .await .map(|_| ()) .map_err(TransportError::from) } @@ -237,19 +238,31 @@ mod tests { Arc::new( Service::new(()) .event("ok") - .handle(move |ctx| { - ok.push(Event::Handled(ctx.message().name().to_string())); - Ok(json!({})) + .handle(move |ctx: &crate::microsvc::Context<()>| { + let ok = ok.clone(); + let name = ctx.message().name().to_string(); + async move { + ok.push(Event::Handled(name)); + Ok(json!({})) + } }) .event("retryable") - .handle(move |ctx| { - retryable.push(Event::Handled(ctx.message().name().to_string())); - Err(HandlerError::Other("infra".into())) + .handle(move |ctx: &crate::microsvc::Context<()>| { + let retryable = retryable.clone(); + let name = ctx.message().name().to_string(); + async move { + retryable.push(Event::Handled(name)); + Err(HandlerError::Other("infra".into())) + } }) .event("permanent") - .handle(move |ctx| { - permanent.push(Event::Handled(ctx.message().name().to_string())); - Err(HandlerError::Rejected("nope".into())) + .handle(move |ctx: &crate::microsvc::Context<()>| { + let permanent = permanent.clone(); + let name = ctx.message().name().to_string(); + async move { + permanent.push(Event::Handled(name)); + Err(HandlerError::Rejected("nope".into())) + } }), ) } From 5e70ac188d8f90e5ff9bda142e71530169b20f91 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 19:54:21 -0500 Subject: [PATCH 36/39] test(microsvc)!: migrate all integration test crates to async handlers Completes the integration half of the async handler switch: all 21 test crates (microsvc, sagas, the gold-standard distributed_read_model matrix, the board, the transport conformance crates, and the ~15 direct-repo crates) now use the async handler + async repo API exclusively: - handlers are `async fn handle(ctx: &Context<'_, D>)` with awaited ctx.repo().get/commit/peek and ctx.repo().outbox(msg).commit(&mut a).await; read-model handlers use workspace_async()/load_async()/commit_async().await. - services build with .queued_async().async_aggregate(); inline handler closures use the `|ctx: &Context| { extract ctx reads; async move { ... } }` form. - test bodies await dispatch and the now-async repo reads. Guards stay synchronous. Assertions and domain logic unchanged. The sync repo trait surface is still present (deleted next); 502 default tests pass, the gold -standard matrix's gated cells compile, http/grpc/sqlite cells pass. Part of [[tasks/async-only-consolidation]] (Phase 3). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/outbox/commit.rs | 11 +- tests/blob_game/main.rs | 12 +- .../handlers/record_seat_reserved.rs | 8 +- .../checkout_saga_service/handlers/start.rs | 5 +- .../checkout_saga_service/mod.rs | 9 +- tests/distributed_read_model/main.rs | 52 ++-- .../projection_service/handlers/checkout.rs | 18 +- .../projection_service/handlers/seat.rs | 18 +- .../query_service/mod.rs | 20 +- .../seat_inventory_service/handlers/add.rs | 5 +- .../handlers/reserve_started_checkout_seat.rs | 8 +- .../seat_inventory_service/mod.rs | 7 +- .../board_service/handlers/board_add_card.rs | 9 +- .../board_service/handlers/board_move_card.rs | 9 +- .../board_service/handlers/board_open.rs | 8 +- .../handlers/board_remove_card.rs | 9 +- .../board_service/mod.rs | 7 +- tests/distributed_read_model_board/main.rs | 32 +- .../projections_service/handlers/board.rs | 11 +- .../projections_service/mod.rs | 9 +- .../query_service/mod.rs | 23 +- tests/enqueue/main.rs | 24 +- tests/event_store/main.rs | 163 +++++++---- tests/kafka_transport/main.rs | 17 +- tests/knative_cloudevents/main.rs | 14 +- tests/microsvc/basic.rs | 77 +++-- tests/microsvc/convention.rs | 63 ++-- tests/microsvc/handlers/counter_create.rs | 8 +- tests/microsvc/handlers/counter_increment.rs | 9 +- tests/microsvc/handlers/mod.rs | 9 +- tests/microsvc/handlers/whoami.rs | 2 +- tests/microsvc/session.rs | 43 ++- tests/microsvc/transport_grpc.rs | 4 +- tests/microsvc/transport_http.rs | 4 +- tests/microsvc/transport_listen.rs | 19 +- tests/microsvc/transport_subscribe.rs | 30 +- tests/nats_transport/main.rs | 17 +- tests/postgres_transport/main.rs | 24 +- tests/rabbitmq_transport/main.rs | 17 +- tests/read_model_session/main.rs | 42 +-- tests/sagas/handlers/inventory/init.rs | 4 +- tests/sagas/handlers/inventory/mod.rs | 9 +- tests/sagas/handlers/inventory/reserve.rs | 7 +- tests/sagas/handlers/orders/complete.rs | 7 +- tests/sagas/handlers/orders/create.rs | 4 +- tests/sagas/handlers/orders/mod.rs | 7 +- tests/sagas/handlers/payments/mod.rs | 9 +- tests/sagas/handlers/payments/process.rs | 4 +- tests/sagas/handlers/saga/mod.rs | 9 +- .../handlers/saga/on_inventory_reserved.rs | 7 +- .../sagas/handlers/saga/on_order_completed.rs | 7 +- tests/sagas/handlers/saga/on_order_created.rs | 7 +- .../handlers/saga/on_payment_succeeded.rs | 7 +- tests/sagas/handlers/saga/start.rs | 4 +- tests/sagas/microsvc_saga.rs | 79 +++-- tests/sagas/orchestration.rs | 180 ++++++------ tests/snapshots/main.rs | 276 ++++++++++++------ tests/sourced/main.rs | 14 +- tests/sourced_enqueue/main.rs | 26 +- tests/sourced_snapshot/main.rs | 62 ++-- tests/todos/main.rs | 235 +++++++++------ tests/transport_conformance/mod.rs | 14 +- tests/upcasting/aggregate.rs | 6 +- tests/upcasting/main.rs | 61 ++-- 64 files changed, 1166 insertions(+), 755 deletions(-) diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 158e052d7..9434eef29 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -51,12 +51,15 @@ where /// Helper returned by [`AsyncAggregateRepository::outbox`] to commit an aggregate /// and an outbox row in the same async transactional batch. -pub struct AsyncOutboxCommit { - repo: AsyncAggregateRepository, +/// +/// Borrows the repository (mirroring the synchronous [`outbox_sync`](AsyncOutboxCommitExt)) +/// so it can be called through `ctx.repo()` inside async handlers. +pub struct AsyncOutboxCommit<'a, R, A> { + repo: &'a AsyncAggregateRepository, message: OutboxMessage, } -impl AsyncOutboxCommit +impl AsyncOutboxCommit<'_, R, A> where R: AsyncTransactionalCommit, A: Aggregate + Send, @@ -74,7 +77,7 @@ where impl AsyncAggregateRepository { /// Attach an outbox message to be committed with the aggregate. - pub fn outbox(self, message: OutboxMessage) -> AsyncOutboxCommit { + pub fn outbox(&self, message: OutboxMessage) -> AsyncOutboxCommit<'_, R, A> { AsyncOutboxCommit { repo: self, message, diff --git a/tests/blob_game/main.rs b/tests/blob_game/main.rs index 8f45495b9..2ea666702 100644 --- a/tests/blob_game/main.rs +++ b/tests/blob_game/main.rs @@ -1,7 +1,7 @@ mod aggregate; use aggregate::{BlobGame, TileState}; -use sourced_rust::{AggregateBuilder, HashMapRepository}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository}; // Tile state shortcuts const P: TileState = TileState::Player; @@ -342,9 +342,9 @@ fn should_work_with_timer_mode() { assert_eq!(game.score(), 4); // Score unchanged from timeout death } -#[test] -fn replay_restores_game_state() { - let repo = HashMapRepository::new().aggregate::(); +#[tokio::test] +async fn replay_restores_game_state() { + let repo = HashMapRepository::new().async_aggregate::(); // Create and play a game let mut game = BlobGame::new(); @@ -364,10 +364,10 @@ fn replay_restores_game_state() { game.right(None).unwrap(); // Commit to repository - repo.commit(&mut game).unwrap(); + repo.commit(&mut game).await.unwrap(); // Retrieve and verify state is restored - let restored = repo.get("game-replay").unwrap().unwrap(); + let restored = repo.get("game-replay").await.unwrap().unwrap(); assert_eq!(restored.score(), 3); assert!(!restored.is_current_level_completed()); assert!(!restored.is_player_dead()); diff --git a/tests/distributed_read_model/checkout_saga_service/handlers/record_seat_reserved.rs b/tests/distributed_read_model/checkout_saga_service/handlers/record_seat_reserved.rs index a3095db93..25992b60b 100644 --- a/tests/distributed_read_model/checkout_saga_service/handlers/record_seat_reserved.rs +++ b/tests/distributed_read_model/checkout_saga_service/handlers/record_seat_reserved.rs @@ -1,6 +1,5 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::SyncOutboxCommitExt; use crate::checkout::{ checkout_event, json_outbox_event, seat_event, SeatReservationCompleted, SeatReserved, @@ -14,11 +13,12 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["checkout_id", "seat_id", "seat_category"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, CheckoutRepo>) -> Result { let msg = ctx.input::()?; let mut saga = ctx .repo() - .get(&msg.checkout_id)? + .get(&msg.checkout_id) + .await? .ok_or_else(|| HandlerError::NotFound(msg.checkout_id.clone()))?; if saga.status == CHECKOUT_SEAT_RESERVED { @@ -36,7 +36,7 @@ pub fn handle(ctx: &Context) -> Result { checkout_event::SEAT_RESERVATION_COMPLETED, &event, )?; - ctx.repo().outbox_sync(out).commit_sync(&mut saga)?; + ctx.repo().outbox(out).commit(&mut saga).await?; Ok(json!({ "checkout_id": msg.checkout_id })) } diff --git a/tests/distributed_read_model/checkout_saga_service/handlers/start.rs b/tests/distributed_read_model/checkout_saga_service/handlers/start.rs index b4326099c..7cc85230a 100644 --- a/tests/distributed_read_model/checkout_saga_service/handlers/start.rs +++ b/tests/distributed_read_model/checkout_saga_service/handlers/start.rs @@ -1,6 +1,5 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::SyncOutboxCommitExt; use crate::checkout::{ checkout_command, checkout_event, json_outbox_event, CheckoutStarted, StartCheckout, @@ -13,7 +12,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["checkout_id", "seat_id", "seat_category"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, CheckoutRepo>) -> Result { let msg = ctx.input::()?; let mut saga = CheckoutSaga::default(); saga.start( @@ -28,7 +27,7 @@ pub fn handle(ctx: &Context) -> Result { seat_category: msg.seat_category.clone(), }; let out = json_outbox_event(&msg.checkout_id, checkout_event::STARTED, &event)?; - ctx.repo().outbox_sync(out).commit_sync(&mut saga)?; + ctx.repo().outbox(out).commit(&mut saga).await?; Ok(json!({ "checkout_id": msg.checkout_id })) } diff --git a/tests/distributed_read_model/checkout_saga_service/mod.rs b/tests/distributed_read_model/checkout_saga_service/mod.rs index 93fa1a19a..79d825706 100644 --- a/tests/distributed_read_model/checkout_saga_service/mod.rs +++ b/tests/distributed_read_model/checkout_saga_service/mod.rs @@ -11,6 +11,11 @@ pub use service::start_grpc_service; #[cfg(feature = "http")] pub use service::start_http_service; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; -pub type CheckoutRepo = AggregateRepository, CheckoutSaga>; +pub type CheckoutRepo = AsyncAggregateRepository< + QueuedRepository, + CheckoutSaga, +>; diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 2dcd2f7da..8605307b6 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -43,18 +43,18 @@ use read_models::{register_schemas, CheckoutView}; use read_models::{CheckoutStepView, SeatView}; use seat_inventory_service::Seat; use serde::Serialize; -use sourced_rust::microsvc::{Service, Session}; +use sourced_rust::microsvc::{Context, Service, Session}; #[cfg(feature = "sqlite")] use sourced_rust::SqliteRepository; -use sourced_rust::{AggregateBuilder, HashMapRepository, InMemoryReadModelStore, Queueable}; use sourced_rust::{ AsyncAggregateBuilder, AsyncCommitBuilderExt, AsyncGetStream, AsyncOutboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, OutboxMessage, ReadModelError, ReadModelWritePlanBuilder, RelationalReadModel, RelationalReadModelIncludes, }; +use sourced_rust::{HashMapRepository, InMemoryReadModelStore, Queueable}; -fn dispatch(service: &Service, command: &str, input: C) +async fn dispatch(service: &Service, command: &str, input: C) where D: Send + Sync + 'static, C: Serialize, @@ -65,6 +65,7 @@ where serde_json::to_value(input).expect("command should encode"), Session::new(), ) + .await .unwrap_or_else(|err| panic!("{command} should dispatch: {err:?}")); } @@ -504,9 +505,10 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let checkout_store = HashMapRepository::new(); let checkout_service = - checkout_saga_service::service(checkout_store.clone().queued().aggregate()); + checkout_saga_service::service(checkout_store.clone().queued_async().async_aggregate()); let seat_store = HashMapRepository::new(); - let seat_service = seat_inventory_service::service(seat_store.clone().queued().aggregate()); + let seat_service = + seat_inventory_service::service(seat_store.clone().queued_async().async_aggregate()); let read_store = InMemoryReadModelStore::new(); register_schemas(&read_store).expect("relational schemas should register"); let projection_svc = projection_service(read_store.clone()); @@ -522,7 +524,8 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { seat_id: "A-7".to_string(), category: "balcony".to_string(), }, - ); + ) + .await; dispatch( &checkout_service, checkout_command::START, @@ -531,7 +534,8 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { seat_id: "A-7".to_string(), seat_category: "balcony".to_string(), }, - ); + ) + .await; // Hop 1: SeatAdded + CheckoutStarted reach the bus; the projection records the // opening state and the seat service reacts to the checkout by reserving. @@ -561,6 +565,7 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let checkout = query_service .checkout_screen("checkout-1") + .await .expect("checkout query should succeed") .expect("checkout should be projected"); assert_eq!(checkout.seat_id, "A-7"); @@ -589,6 +594,7 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let seat = query_service .seat("A-7") + .await .expect("seat query should succeed") .expect("seat should be projected"); assert_eq!(seat.status, SEAT_RESERVED); @@ -596,9 +602,10 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let checkout_saga = checkout_store .clone() - .queued() - .aggregate::() + .queued_async() + .async_aggregate::() .peek("checkout-1") + .await .unwrap() .unwrap(); assert_eq!(checkout_saga.status, CHECKOUT_SEAT_RESERVED); @@ -606,9 +613,10 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let seat = seat_store .clone() - .queued() - .aggregate::() + .queued_async() + .async_aggregate::() .peek("A-7") + .await .unwrap() .unwrap(); assert_eq!(seat.status, SEAT_RESERVED); @@ -693,7 +701,7 @@ async fn async_postgres_checkout_flow_projects_relational_read_models() { async fn checkout_commands_can_be_http_service() { let checkout_store = HashMapRepository::new(); let checkout_service = - checkout_saga_service::service(checkout_store.clone().queued().aggregate()); + checkout_saga_service::service(checkout_store.clone().queued_async().async_aggregate()); let base = checkout_saga_service::start_http_service(checkout_service.clone()).await; let client = reqwest::Client::new(); @@ -712,6 +720,7 @@ async fn checkout_commands_can_be_http_service() { let saga = checkout_service .repo() .peek("checkout-http") + .await .expect("HTTP write-side load should succeed") .expect("HTTP write-side checkout should exist"); assert_eq!(saga.status, checkout::CHECKOUT_STARTED); @@ -722,7 +731,7 @@ async fn checkout_commands_can_be_http_service() { async fn checkout_commands_can_be_grpc_service() { let checkout_store = HashMapRepository::new(); let checkout_service = - checkout_saga_service::service(checkout_store.clone().queued().aggregate()); + checkout_saga_service::service(checkout_store.clone().queued_async().async_aggregate()); let mut client = checkout_saga_service::start_grpc_service(checkout_service.clone()).await; let started = client @@ -744,6 +753,7 @@ async fn checkout_commands_can_be_grpc_service() { let saga = checkout_service .repo() .peek("checkout-grpc") + .await .expect("gRPC write-side load should succeed") .expect("gRPC write-side checkout should exist"); assert_eq!(saga.status, checkout::CHECKOUT_STARTED); @@ -794,24 +804,24 @@ fn build_collector() -> (StdArc>, Collected) { ); let service = Service::new(()) .event(seat_event::ADDED) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { record_message(&c1, ctx.message()); - Ok(serde_json::Value::Null) + async { Ok(serde_json::Value::Null) } }) .event(checkout_event::STARTED) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { record_message(&c2, ctx.message()); - Ok(serde_json::Value::Null) + async { Ok(serde_json::Value::Null) } }) .event(seat_event::RESERVED) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { record_message(&c3, ctx.message()); - Ok(serde_json::Value::Null) + async { Ok(serde_json::Value::Null) } }) .event(checkout_event::SEAT_RESERVATION_COMPLETED) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { record_message(&c4, ctx.message()); - Ok(serde_json::Value::Null) + async { Ok(serde_json::Value::Null) } }); (StdArc::new(service), collected) } diff --git a/tests/distributed_read_model/projection_service/handlers/checkout.rs b/tests/distributed_read_model/projection_service/handlers/checkout.rs index 65c2cfc2f..3892737de 100644 --- a/tests/distributed_read_model/projection_service/handlers/checkout.rs +++ b/tests/distributed_read_model/projection_service/handlers/checkout.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::ReadModelWorkspaceExt; +use sourced_rust::AsyncReadModelWorkspaceExt; use crate::checkout::{ checkout_event, CheckoutStarted, SeatReservationCompleted, CHECKOUT_SEAT_RESERVED, @@ -18,7 +18,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.message().id().is_some() } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, ProjectionDependencies>) -> Result { match ctx.message().name() { checkout_event::STARTED => { let msg: CheckoutStarted = serde_json::from_slice(ctx.message().payload()) @@ -34,12 +34,15 @@ pub fn handle(ctx: &Context) -> Result { let msg: SeatReservationCompleted = serde_json::from_slice(ctx.message().payload()) @@ -61,12 +64,15 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), } diff --git a/tests/distributed_read_model/projection_service/handlers/seat.rs b/tests/distributed_read_model/projection_service/handlers/seat.rs index 128b1aa98..117a443e4 100644 --- a/tests/distributed_read_model/projection_service/handlers/seat.rs +++ b/tests/distributed_read_model/projection_service/handlers/seat.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::ReadModelWorkspaceExt; +use sourced_rust::AsyncReadModelWorkspaceExt; use crate::checkout::{seat_event, SeatAdded, SeatReserved, SEAT_AVAILABLE, SEAT_RESERVED}; use crate::projection_service::ProjectionDependencies; @@ -12,7 +12,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.message().id().is_some() } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, ProjectionDependencies>) -> Result { match ctx.message().name() { seat_event::ADDED => { let msg: SeatAdded = serde_json::from_slice(ctx.message().payload()) @@ -24,9 +24,12 @@ pub fn handle(ctx: &Context) -> Result { let msg: SeatReserved = serde_json::from_slice(ctx.message().payload()) @@ -43,10 +46,13 @@ pub fn handle(ctx: &Context) -> Result return Err(HandlerError::UnknownCommand(other.to_string())), } diff --git a/tests/distributed_read_model/query_service/mod.rs b/tests/distributed_read_model/query_service/mod.rs index 5050d9e0a..5b94502c1 100644 --- a/tests/distributed_read_model/query_service/mod.rs +++ b/tests/distributed_read_model/query_service/mod.rs @@ -2,7 +2,7 @@ //! projected relational tables through primary-key loads plus explicit //! relationship includes. -use sourced_rust::{InMemoryReadModelStore, ReadModelError, ReadModelWorkspaceExt}; +use sourced_rust::{AsyncReadModelWorkspaceExt, InMemoryReadModelStore, ReadModelError}; use crate::read_models::{checkout_key, seat_key, CheckoutView, SeatView}; @@ -17,24 +17,26 @@ impl CheckoutQueryService { } /// Load the checkout screen with its audit steps and current seat row. - pub fn checkout_screen( + pub async fn checkout_screen( &self, checkout_id: &str, ) -> Result, ReadModelError> { - let mut session = self.store.workspace(); + let mut session = self.store.workspace_async(); Ok(session - .load::(checkout_key(checkout_id)) + .load_async::(checkout_key(checkout_id)) .include("steps") .include("seat") - .one()? + .one() + .await? .map(|view| view.data)) } - pub fn seat(&self, seat_id: &str) -> Result, ReadModelError> { - let mut session = self.store.workspace(); + pub async fn seat(&self, seat_id: &str) -> Result, ReadModelError> { + let mut session = self.store.workspace_async(); Ok(session - .load::(seat_key(seat_id)) - .one()? + .load_async::(seat_key(seat_id)) + .one() + .await? .map(|view| view.data)) } } diff --git a/tests/distributed_read_model/seat_inventory_service/handlers/add.rs b/tests/distributed_read_model/seat_inventory_service/handlers/add.rs index 1e91f01e5..ca0d1d575 100644 --- a/tests/distributed_read_model/seat_inventory_service/handlers/add.rs +++ b/tests/distributed_read_model/seat_inventory_service/handlers/add.rs @@ -1,6 +1,5 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::SyncOutboxCommitExt; use crate::checkout::{json_outbox_event, seat_command, seat_event, AddSeat, SeatAdded}; use crate::seat_inventory_service::{Seat, SeatRepo}; @@ -11,7 +10,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["seat_id", "category"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, SeatRepo>) -> Result { let msg = ctx.input::()?; let mut seat = Seat::default(); seat.add(msg.seat_id.clone(), msg.category.clone())?; @@ -21,7 +20,7 @@ pub fn handle(ctx: &Context) -> Result { category: msg.category.clone(), }; let out = json_outbox_event(&msg.seat_id, seat_event::ADDED, &event)?; - ctx.repo().outbox_sync(out).commit_sync(&mut seat)?; + ctx.repo().outbox(out).commit(&mut seat).await?; Ok(json!({ "seat_id": msg.seat_id })) } diff --git a/tests/distributed_read_model/seat_inventory_service/handlers/reserve_started_checkout_seat.rs b/tests/distributed_read_model/seat_inventory_service/handlers/reserve_started_checkout_seat.rs index cc61ebf76..cf467be64 100644 --- a/tests/distributed_read_model/seat_inventory_service/handlers/reserve_started_checkout_seat.rs +++ b/tests/distributed_read_model/seat_inventory_service/handlers/reserve_started_checkout_seat.rs @@ -1,6 +1,5 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::SyncOutboxCommitExt; use crate::checkout::{ checkout_event, json_outbox_event, seat_event, CheckoutStarted, SeatReserved, SEAT_AVAILABLE, @@ -14,11 +13,12 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["checkout_id", "seat_id", "seat_category"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, SeatRepo>) -> Result { let msg = ctx.input::()?; let mut seat = ctx .repo() - .get(&msg.seat_id)? + .get(&msg.seat_id) + .await? .ok_or_else(|| HandlerError::NotFound(msg.seat_id.clone()))?; if seat.checkout_id == msg.checkout_id && seat.status == SEAT_RESERVED { @@ -46,7 +46,7 @@ pub fn handle(ctx: &Context) -> Result { seat_category: msg.seat_category.clone(), }; let out = json_outbox_event(&msg.checkout_id, seat_event::RESERVED, &event)?; - ctx.repo().outbox_sync(out).commit_sync(&mut seat)?; + ctx.repo().outbox(out).commit(&mut seat).await?; Ok(json!({ "seat_id": msg.seat_id })) } diff --git a/tests/distributed_read_model/seat_inventory_service/mod.rs b/tests/distributed_read_model/seat_inventory_service/mod.rs index 54617258a..352b8a15f 100644 --- a/tests/distributed_read_model/seat_inventory_service/mod.rs +++ b/tests/distributed_read_model/seat_inventory_service/mod.rs @@ -6,6 +6,9 @@ pub mod models; pub use models::Seat; pub use service::service; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; -pub type SeatRepo = AggregateRepository, Seat>; +pub type SeatRepo = + AsyncAggregateRepository, Seat>; diff --git a/tests/distributed_read_model_board/board_service/handlers/board_add_card.rs b/tests/distributed_read_model_board/board_service/handlers/board_add_card.rs index f5ec34801..ee94aebf8 100644 --- a/tests/distributed_read_model_board/board_service/handlers/board_add_card.rs +++ b/tests/distributed_read_model_board/board_service/handlers/board_add_card.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use crate::board_service::{AddCard, Board, BoardRepo}; @@ -10,12 +10,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id", "card_id", "column", "title"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, BoardRepo>) -> Result { let input = ctx.input::()?; let mut board: Board = ctx .repo() - .get(&input.id)? + .get(&input.id) + .await? .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; board.add_card( input.card_id.clone(), @@ -26,7 +27,7 @@ pub fn handle(ctx: &Context) -> Result { )?; let outbox = OutboxMessage::domain_event("board.card_added", &board)?; - ctx.repo().outbox_sync(outbox).commit_sync(&mut board)?; + ctx.repo().outbox(outbox).commit(&mut board).await?; Ok(json!({ "id": input.id, "card_id": input.card_id })) } diff --git a/tests/distributed_read_model_board/board_service/handlers/board_move_card.rs b/tests/distributed_read_model_board/board_service/handlers/board_move_card.rs index f2d61b329..1b8308276 100644 --- a/tests/distributed_read_model_board/board_service/handlers/board_move_card.rs +++ b/tests/distributed_read_model_board/board_service/handlers/board_move_card.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use crate::board_service::{Board, BoardRepo, MoveCard}; @@ -10,17 +10,18 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id", "card_id", "column"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, BoardRepo>) -> Result { let input = ctx.input::()?; let mut board: Board = ctx .repo() - .get(&input.id)? + .get(&input.id) + .await? .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; board.move_card(input.card_id.clone(), input.column.clone())?; let outbox = OutboxMessage::domain_event("board.card_moved", &board)?; - ctx.repo().outbox_sync(outbox).commit_sync(&mut board)?; + ctx.repo().outbox(outbox).commit(&mut board).await?; Ok(json!({ "id": input.id, "card_id": input.card_id, "column": input.column })) } diff --git a/tests/distributed_read_model_board/board_service/handlers/board_open.rs b/tests/distributed_read_model_board/board_service/handlers/board_open.rs index f6fe032d3..c541c39bf 100644 --- a/tests/distributed_read_model_board/board_service/handlers/board_open.rs +++ b/tests/distributed_read_model_board/board_service/handlers/board_open.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use crate::board_service::{Board, BoardRepo, OpenBoard}; @@ -10,9 +10,9 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id", "name"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, BoardRepo>) -> Result { let input = ctx.input::()?; - if ctx.repo().peek(&input.id)?.is_some() { + if ctx.repo().peek(&input.id).await?.is_some() { return Err(HandlerError::Rejected(format!( "board {} already exists", input.id @@ -23,7 +23,7 @@ pub fn handle(ctx: &Context) -> Result { board.open(input.id.clone(), input.name.clone())?; let outbox = OutboxMessage::domain_event("board.opened", &board)?; - ctx.repo().outbox_sync(outbox).commit_sync(&mut board)?; + ctx.repo().outbox(outbox).commit(&mut board).await?; Ok(json!({ "id": input.id })) } diff --git a/tests/distributed_read_model_board/board_service/handlers/board_remove_card.rs b/tests/distributed_read_model_board/board_service/handlers/board_remove_card.rs index 5a498119a..9902d8ce2 100644 --- a/tests/distributed_read_model_board/board_service/handlers/board_remove_card.rs +++ b/tests/distributed_read_model_board/board_service/handlers/board_remove_card.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use crate::board_service::{Board, BoardRepo, RemoveCard}; @@ -10,17 +10,18 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id", "card_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, BoardRepo>) -> Result { let input = ctx.input::()?; let mut board: Board = ctx .repo() - .get(&input.id)? + .get(&input.id) + .await? .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; board.remove_card(input.card_id.clone())?; let outbox = OutboxMessage::domain_event("board.card_removed", &board)?; - ctx.repo().outbox_sync(outbox).commit_sync(&mut board)?; + ctx.repo().outbox(outbox).commit(&mut board).await?; Ok(json!({ "id": input.id, "card_id": input.card_id })) } diff --git a/tests/distributed_read_model_board/board_service/mod.rs b/tests/distributed_read_model_board/board_service/mod.rs index b8b9d4d8c..66eebeb7a 100644 --- a/tests/distributed_read_model_board/board_service/mod.rs +++ b/tests/distributed_read_model_board/board_service/mod.rs @@ -6,9 +6,12 @@ pub mod models; mod handlers; mod service; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; pub use models::{AddCard, Board, BoardSnapshot, MoveCard, OpenBoard, RemoveCard}; pub use service::model_service; -pub type BoardRepo = AggregateRepository, Board>; +pub type BoardRepo = + AsyncAggregateRepository, Board>; diff --git a/tests/distributed_read_model_board/main.rs b/tests/distributed_read_model_board/main.rs index af4ea702d..6f37abe88 100644 --- a/tests/distributed_read_model_board/main.rs +++ b/tests/distributed_read_model_board/main.rs @@ -24,11 +24,11 @@ use serde::Serialize; use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; use sourced_rust::{ - AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, - InMemoryReadModelStore, OutboxClaimRef, Queueable, + AsyncAggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, + HashMapRepository, InMemoryReadModelStore, OutboxClaimRef, Queueable, }; -fn dispatch(service: &Service, command: &str, input: C) +async fn dispatch(service: &Service, command: &str, input: C) where D: Send + Sync + 'static, C: Serialize, @@ -39,6 +39,7 @@ where serde_json::to_value(input).expect("command should encode"), Session::new(), ) + .await .unwrap_or_else(|err| panic!("{command} should dispatch: {err:?}")); } @@ -78,7 +79,7 @@ async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) async fn board_service_feeds_a_normalized_card_read_model() { let board_store = HashMapRepository::new(); let board_outbox = board_store.outbox_store(); - let board_service = board_service::model_service(board_store.queued().aggregate()); + let board_service = board_service::model_service(board_store.queued_async().async_aggregate()); let read_store = InMemoryReadModelStore::new(); register_schemas(&read_store).expect("relational schemas should register"); @@ -92,7 +93,8 @@ async fn board_service_feeds_a_normalized_card_read_model() { id: "board-1".to_string(), name: "Roadmap".to_string(), }, - ); + ) + .await; dispatch( &board_service, "board.add_card", @@ -104,7 +106,8 @@ async fn board_service_feeds_a_normalized_card_read_model() { labels: vec!["design".to_string()], assignee: Some("ada".to_string()), }, - ); + ) + .await; dispatch( &board_service, "board.add_card", @@ -116,7 +119,8 @@ async fn board_service_feeds_a_normalized_card_read_model() { labels: vec!["code".to_string()], assignee: None, }, - ); + ) + .await; dispatch( &board_service, "board.move_card", @@ -125,7 +129,8 @@ async fn board_service_feeds_a_normalized_card_read_model() { card_id: "card-spec".to_string(), column: "doing".to_string(), }, - ); + ) + .await; dispatch( &board_service, "board.remove_card", @@ -133,7 +138,8 @@ async fn board_service_feeds_a_normalized_card_read_model() { id: "board-1".to_string(), card_id: "card-impl".to_string(), }, - ); + ) + .await; // Forward the board's outbox events onto the bus, then drain them into the // projection in one pass. The projection's monotonic `source_version` guard @@ -145,7 +151,9 @@ async fn board_service_feeds_a_normalized_card_read_model() { .await .expect("projection should drain the board events"); - let board = load_board(&read_store, "board-1").expect("board should be projected"); + let board = load_board(&read_store, "board-1") + .await + .expect("board should be projected"); assert_eq!(board.name, "Roadmap"); assert_eq!(board.cards.len(), 1, "removed card should be deleted"); @@ -159,6 +167,7 @@ async fn board_service_feeds_a_normalized_card_read_model() { // belongs_to include resolves the card's board. let card_with_board = query_service .card_with_board("board-1", "card-spec") + .await .expect("query should succeed") .expect("card should exist"); let parent = card_with_board @@ -170,6 +179,7 @@ async fn board_service_feeds_a_normalized_card_read_model() { // The removed card's row is gone. assert!(query_service .board_with_cards("board-1") + .await .expect("query should succeed") .expect("board should exist") .cards @@ -177,12 +187,14 @@ async fn board_service_feeds_a_normalized_card_read_model() { .all(|card| card.card_id != "card-impl")); assert!(query_service .card_with_board("board-1", "card-impl") + .await .expect("query should succeed") .is_none()); let write_side = board_service .repo() .peek("board-1") + .await .expect("write-side load should succeed") .expect("write-side board should exist"); assert_eq!(write_side.cards.len(), 1); diff --git a/tests/distributed_read_model_board/projections_service/handlers/board.rs b/tests/distributed_read_model_board/projections_service/handlers/board.rs index 002f163e4..da87a4528 100644 --- a/tests/distributed_read_model_board/projections_service/handlers/board.rs +++ b/tests/distributed_read_model_board/projections_service/handlers/board.rs @@ -5,7 +5,7 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{BitcodePayloadCodec, PayloadCodec, ReadModelWorkspaceExt}; +use sourced_rust::{AsyncReadModelWorkspaceExt, BitcodePayloadCodec, PayloadCodec}; use crate::board_service::BoardSnapshot; use crate::projections_service::{read_model_error, ProjectionDependencies}; @@ -22,7 +22,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.message().id().is_some() } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, ProjectionDependencies>) -> Result { let message_id = ctx .message() .id() @@ -32,11 +32,12 @@ pub fn handle(ctx: &Context) -> Result(board_key(&updated_view.board_id)) + .load_async::(board_key(&updated_view.board_id)) .include("cards") .one() + .await .map_err(read_model_error)?; match existing { @@ -52,7 +53,7 @@ pub fn handle(ctx: &Context) -> Result HandlerError { /// Load the projected board (with its cards) from the read store. After the /// bus has been drained into the projection service, the board reflects every /// processed event. -pub fn load_board(store: &InMemoryReadModelStore, board_id: &str) -> Option { +pub async fn load_board(store: &InMemoryReadModelStore, board_id: &str) -> Option { store - .workspace() - .load::(board_key(board_id)) + .workspace_async() + .load_async::(board_key(board_id)) .include("cards") .one() + .await .expect("board load should succeed") .map(|view| view.data) } diff --git a/tests/distributed_read_model_board/query_service/mod.rs b/tests/distributed_read_model_board/query_service/mod.rs index 94726353d..bd6f6e59c 100644 --- a/tests/distributed_read_model_board/query_service/mod.rs +++ b/tests/distributed_read_model_board/query_service/mod.rs @@ -1,7 +1,7 @@ //! Read-only query service for the board read model. Primary-key loads plus //! `has_many` / `belongs_to` relationship includes. -use sourced_rust::{InMemoryReadModelStore, ReadModelError, ReadModelWorkspaceExt}; +use sourced_rust::{AsyncReadModelWorkspaceExt, InMemoryReadModelStore, ReadModelError}; use crate::read_models::{board_key, card_key, BoardView, CardView}; @@ -16,26 +16,31 @@ impl BoardQueryService { } /// Load a board with its cards (`has_many` include). - pub fn board_with_cards(&self, board_id: &str) -> Result, ReadModelError> { - let mut session = self.store.workspace(); + pub async fn board_with_cards( + &self, + board_id: &str, + ) -> Result, ReadModelError> { + let mut session = self.store.workspace_async(); Ok(session - .load::(board_key(board_id)) + .load_async::(board_key(board_id)) .include("cards") - .one()? + .one() + .await? .map(|view| view.data)) } /// Load one card with its board (`belongs_to` include). - pub fn card_with_board( + pub async fn card_with_board( &self, board_id: &str, card_id: &str, ) -> Result, ReadModelError> { - let mut session = self.store.workspace(); + let mut session = self.store.workspace_async(); Ok(session - .load::(card_key(board_id, card_id)) + .load_async::(card_key(board_id, card_id)) .include("board") - .one()? + .one() + .await? .map(|view| view.data)) } } diff --git a/tests/enqueue/main.rs b/tests/enqueue/main.rs index 5e9417a66..c3dc7720f 100644 --- a/tests/enqueue/main.rs +++ b/tests/enqueue/main.rs @@ -1,6 +1,6 @@ mod aggregate; -use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, Queueable}; use std::sync::mpsc; use std::time::Duration; @@ -211,9 +211,11 @@ fn digest_and_enqueue_guards_stay_in_sync() { // #[enqueue] with repository commit + replay // ============================================================================= -#[test] -fn enqueue_events_survive_commit_and_emit_after() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn enqueue_events_survive_commit_and_emit_after() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut order = Order::default(); order.create("order-1".into(), "alice".into()).unwrap(); @@ -222,7 +224,7 @@ fn enqueue_events_survive_commit_and_emit_after() { // Events queued before commit assert_eq!(order.emitter.queued_len(), 2); - repo.commit(&mut order).unwrap(); + repo.commit(&mut order).await.unwrap(); // Events still queued after commit — emit is explicit assert_eq!(order.emitter.queued_len(), 2); @@ -239,19 +241,21 @@ fn enqueue_events_survive_commit_and_emit_after() { assert_eq!(order.emitter.queued_len(), 0); } -#[test] -fn replay_does_not_enqueue_events() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn replay_does_not_enqueue_events() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut order = Order::default(); order.create("order-1".into(), "alice".into()).unwrap(); order.confirm().unwrap(); order.emitter.emit_queued(); - repo.commit(&mut order).unwrap(); + repo.commit(&mut order).await.unwrap(); // Load from repo — replays digest events but should NOT re-enqueue - let loaded = repo.get("order-1").unwrap().unwrap(); + let loaded = repo.get("order-1").await.unwrap().unwrap(); assert_eq!(loaded.emitter.queued_len(), 0); assert_eq!(loaded.status, "confirmed"); assert_eq!(loaded.entity.version(), 2); diff --git a/tests/event_store/main.rs b/tests/event_store/main.rs index 570bcf499..c28fdad2e 100644 --- a/tests/event_store/main.rs +++ b/tests/event_store/main.rs @@ -1,4 +1,50 @@ -use sourced_rust::{Commit, Entity, GetOne, HashMapRepository}; +use sourced_rust::{ + AsyncCommitBatch, AsyncGetStream, AsyncStreamWrite, AsyncTransactionalCommit, Entity, + HashMapRepository, StreamIdentity, +}; + +/// Fixed aggregate type used to key every event stream in this crate. +/// +/// The synchronous, id-only repository API (`Commit`/`GetOne`) is being removed, +/// so these event-store semantics tests now run on the async stream API +/// (`get_stream`/`commit_batch_async`). The async path keys streams by full +/// `StreamIdentity` (aggregate type + id), so every entity is committed and +/// loaded under this single aggregate type to mirror the old id-only behavior. +const AGGREGATE_TYPE: &str = "event_store_test"; + +fn identity(id: &str) -> StreamIdentity { + StreamIdentity::new(AGGREGATE_TYPE, id).unwrap() +} + +/// Async equivalent of the old `repo.get_one(id)`. +async fn get_one(repo: &HashMapRepository, id: &str) -> Option { + repo.get_stream(&identity(id)).await.unwrap() +} + +/// Async equivalent of the old `repo.commit(&mut entity)` for a single entity. +async fn commit_one( + repo: &HashMapRepository, + entity: &mut Entity, +) -> Result<(), sourced_rust::RepositoryError> { + let id = entity.id().to_string(); + let stream = AsyncStreamWrite::new(identity(&id), entity); + repo.commit_batch_async(AsyncCommitBatch::new(vec![stream])) + .await +} + +/// Async equivalent of the old `repo.commit(&mut [&mut a, &mut b])` for many entities. +async fn commit_many( + repo: &HashMapRepository, + entities: &mut [&mut Entity], +) -> Result<(), sourced_rust::RepositoryError> { + let mut streams = Vec::with_capacity(entities.len()); + for entity in entities.iter_mut() { + let id = entity.id().to_string(); + streams.push(AsyncStreamWrite::new(identity(&id), entity)); + } + repo.commit_batch_async(AsyncCommitBatch::new(streams)) + .await +} // --- Event Accumulation --- @@ -15,29 +61,29 @@ fn digest_adds_events_with_correct_sequences() { assert_eq!(entity.events()[2].sequence, 3); } -#[test] -fn multiple_load_modify_commit_cycles_accumulate_all_events() { +#[tokio::test] +async fn multiple_load_modify_commit_cycles_accumulate_all_events() { let repo = HashMapRepository::new(); // Cycle 1: create and commit let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Cycle 2: load, modify, commit - let mut entity = repo.get_one("e1").unwrap().unwrap(); + let mut entity = get_one(&repo, "e1").await.unwrap(); assert_eq!(entity.events().len(), 1); entity.digest("Updated", &"v2").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Cycle 3: load, modify, commit - let mut entity = repo.get_one("e1").unwrap().unwrap(); + let mut entity = get_one(&repo, "e1").await.unwrap(); assert_eq!(entity.events().len(), 2); entity.digest("Updated", &"v3").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Verify all events accumulated - let entity = repo.get_one("e1").unwrap().unwrap(); + let entity = get_one(&repo, "e1").await.unwrap(); assert_eq!(entity.events().len(), 3); assert_eq!(entity.events()[0].event_name, "Created"); assert_eq!(entity.events()[1].event_name, "Updated"); @@ -47,77 +93,77 @@ fn multiple_load_modify_commit_cycles_accumulate_all_events() { // --- Append Semantics --- -#[test] -fn commit_appends_only_new_events() { +#[tokio::test] +async fn commit_appends_only_new_events() { let repo = HashMapRepository::new(); let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Reload and add one more event - let mut entity = repo.get_one("e1").unwrap().unwrap(); + let mut entity = get_one(&repo, "e1").await.unwrap(); entity.digest("Updated", &"v2").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Verify via get_one: exactly 2 events - let loaded = repo.get_one("e1").unwrap().unwrap(); + let loaded = get_one(&repo, "e1").await.unwrap(); assert_eq!(loaded.events().len(), 2); assert_eq!(loaded.events()[0].event_name, "Created"); assert_eq!(loaded.events()[1].event_name, "Updated"); } -#[test] -fn empty_commit_is_idempotent() { +#[tokio::test] +async fn empty_commit_is_idempotent() { let repo = HashMapRepository::new(); // Create initial state let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Load and commit without changes - let mut entity = repo.get_one("e1").unwrap().unwrap(); + let mut entity = get_one(&repo, "e1").await.unwrap(); assert!(entity.new_events().is_empty()); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Storage unchanged - let loaded = repo.get_one("e1").unwrap().unwrap(); + let loaded = get_one(&repo, "e1").await.unwrap(); assert_eq!(loaded.events().len(), 1); } -#[test] -fn events_grow_monotonically() { +#[tokio::test] +async fn events_grow_monotonically() { let repo = HashMapRepository::new(); for i in 0..5 { let mut entity = if i == 0 { Entity::with_id("e1") } else { - repo.get_one("e1").unwrap().unwrap() + get_one(&repo, "e1").await.unwrap() }; entity.digest("Event", &format!("v{}", i)).unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); - let loaded = repo.get_one("e1").unwrap().unwrap(); + let loaded = get_one(&repo, "e1").await.unwrap(); assert_eq!(loaded.events().len(), i + 1); } } // --- Optimistic Concurrency --- -#[test] -fn concurrent_writes_detected() { +#[tokio::test] +async fn concurrent_writes_detected() { let repo = HashMapRepository::new(); // Create initial state let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); // Two readers load the same version - let mut reader1 = repo.get_one("e1").unwrap().unwrap(); - let mut reader2 = repo.get_one("e1").unwrap().unwrap(); + let mut reader1 = get_one(&repo, "e1").await.unwrap(); + let mut reader2 = get_one(&repo, "e1").await.unwrap(); assert_eq!(reader1.committed_version(), 1); assert_eq!(reader2.committed_version(), 1); @@ -127,17 +173,19 @@ fn concurrent_writes_detected() { reader2.digest("UpdatedByR2", &"r2").unwrap(); // First commit succeeds - repo.commit(&mut reader1).unwrap(); + commit_one(&repo, &mut reader1).await.unwrap(); // Second commit fails with ConcurrentWrite - let err = repo.commit(&mut reader2).unwrap_err(); + let err = commit_one(&repo, &mut reader2).await.unwrap_err(); match err { sourced_rust::RepositoryError::ConcurrentWrite { id, expected, actual, } => { - assert_eq!(id, "e1"); + // Async stream commits key by full stream identity, so the + // conflict id is reported as ":". + assert_eq!(id, format!("{}:e1", AGGREGATE_TYPE)); assert_eq!(expected, 1); // reader2 loaded at version 1 assert_eq!(actual, 2); // storage now has 2 events } @@ -145,8 +193,8 @@ fn concurrent_writes_detected() { } } -#[test] -fn partial_conflict_rolls_back_entire_commit() { +#[tokio::test] +async fn partial_conflict_rolls_back_entire_commit() { let repo = HashMapRepository::new(); // Create two entities @@ -154,33 +202,36 @@ fn partial_conflict_rolls_back_entire_commit() { e1.digest("Created", &"v1").unwrap(); let mut e2 = Entity::with_id("e2"); e2.digest("Created", &"v1").unwrap(); - repo.commit(&mut [&mut e1, &mut e2]).unwrap(); + commit_many(&repo, &mut [&mut e1, &mut e2]).await.unwrap(); // Load both entities at version 1 - let mut e1_a = repo.get_one("e1").unwrap().unwrap(); - let mut e2_a = repo.get_one("e2").unwrap().unwrap(); + let mut e1_a = get_one(&repo, "e1").await.unwrap(); + let mut e2_a = get_one(&repo, "e2").await.unwrap(); // Concurrently modify e2 from another "session" - let mut e2_b = repo.get_one("e2").unwrap().unwrap(); + let mut e2_b = get_one(&repo, "e2").await.unwrap(); e2_b.digest("Conflict", &"b").unwrap(); - repo.commit(&mut e2_b).unwrap(); + commit_one(&repo, &mut e2_b).await.unwrap(); // Try to commit both e1_a and e2_a together // e1 would be fine, but e2 has a version conflict e1_a.digest("Update", &"a").unwrap(); e2_a.digest("Update", &"a").unwrap(); - let err = repo.commit(&mut [&mut e1_a, &mut e2_a]).unwrap_err(); + let err = commit_many(&repo, &mut [&mut e1_a, &mut e2_a]) + .await + .unwrap_err(); match err { sourced_rust::RepositoryError::ConcurrentWrite { id, .. } => { - assert_eq!(id, "e2"); + // Async stream commits report the conflicting stream by full identity. + assert_eq!(id, format!("{}:e2", AGGREGATE_TYPE)); } other => panic!("expected ConcurrentWrite, got: {:?}", other), } // e1 should NOT have been modified (atomic rollback - phase 1 validates all before writing) - let e1_loaded = repo.get_one("e1").unwrap().unwrap(); + let e1_loaded = get_one(&repo, "e1").await.unwrap(); assert_eq!(e1_loaded.events().len(), 1); - let e2_loaded = repo.get_one("e2").unwrap().unwrap(); + let e2_loaded = get_one(&repo, "e2").await.unwrap(); assert_eq!(e2_loaded.events().len(), 2); } @@ -194,31 +245,31 @@ fn new_entity_has_zero_versions() { assert!(entity.new_events().is_empty()); } -#[test] -fn load_from_history_sets_committed_version() { +#[tokio::test] +async fn load_from_history_sets_committed_version() { let repo = HashMapRepository::new(); let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); entity.digest("Updated", &"v2").unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); - let loaded = repo.get_one("e1").unwrap().unwrap(); + let loaded = get_one(&repo, "e1").await.unwrap(); assert_eq!(loaded.committed_version(), 2); assert_eq!(loaded.snapshot_version(), 0); assert_eq!(loaded.version(), 2); assert!(loaded.new_events().is_empty()); } -#[test] -fn commit_updates_committed_version() { +#[tokio::test] +async fn commit_updates_committed_version() { let repo = HashMapRepository::new(); let mut entity = Entity::with_id("e1"); entity.digest("Created", &"v1").unwrap(); assert_eq!(entity.committed_version(), 0); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); assert_eq!(entity.committed_version(), 1); assert_eq!(entity.snapshot_version(), 0); assert!(entity.new_events().is_empty()); @@ -227,14 +278,14 @@ fn commit_updates_committed_version() { entity.digest("Updated", &"v2").unwrap(); assert_eq!(entity.new_events().len(), 1); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); assert_eq!(entity.committed_version(), 2); assert_eq!(entity.snapshot_version(), 0); assert!(entity.new_events().is_empty()); } -#[test] -fn new_events_returns_only_uncommitted() { +#[tokio::test] +async fn new_events_returns_only_uncommitted() { let repo = HashMapRepository::new(); let mut entity = Entity::with_id("e1"); @@ -242,7 +293,7 @@ fn new_events_returns_only_uncommitted() { entity.digest("e2", &"b").unwrap(); assert_eq!(entity.new_events().len(), 2); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); assert!(entity.new_events().is_empty()); entity.digest("e3", &"c").unwrap(); diff --git a/tests/kafka_transport/main.rs b/tests/kafka_transport/main.rs index a801f2703..df1304f99 100644 --- a/tests/kafka_transport/main.rs +++ b/tests/kafka_transport/main.rs @@ -14,7 +14,7 @@ use sourced_rust::microsvc::transport::{ run_source, AsyncMessagePublisher, Bus, BusConsumer, KafkaBus, KafkaPublisher, KafkaSource, RunOptions, }; -use sourced_rust::microsvc::{Message, MessageKind, Service}; +use sourced_rust::microsvc::{Context, Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); @@ -27,11 +27,11 @@ fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), }; - Arc::new(registered.handle(move |ctx| { + Arc::new(registered.handle(move |ctx: &Context<()>| { rec.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } })) } @@ -80,11 +80,11 @@ async fn publish_then_consume_round_trips_through_kafka() { let service = Arc::new( Service::new(()) .event(Box::leak(topic.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { h.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } }), ); run_source(service, source, RunOptions::idempotent()) @@ -125,14 +125,15 @@ async fn message_id_and_metadata_survive_the_round_trip() { let service = Arc::new( Service::new(()) .event(Box::leak(topic.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { let m = ctx.message(); - *o.lock().unwrap() = Some(( + let recorded = Some(( m.id().map(str::to_string), m.correlation_id().map(str::to_string), m.payload().to_vec(), )); - Ok(json!({})) + *o.lock().unwrap() = recorded; + async move { Ok(json!({})) } }), ); run_source(service, source, RunOptions::idempotent()) diff --git a/tests/knative_cloudevents/main.rs b/tests/knative_cloudevents/main.rs index cf4884f30..74a1958a4 100644 --- a/tests/knative_cloudevents/main.rs +++ b/tests/knative_cloudevents/main.rs @@ -9,7 +9,9 @@ use std::sync::{Arc, Mutex}; use serde_json::json; use sourced_rust::microsvc::transport::{cloud_events_router, Bus, KnativeBus}; -use sourced_rust::microsvc::{HandlerError, Message, MessageKind, Service, SubscriptionPlan}; +use sourced_rust::microsvc::{ + Context, HandlerError, Message, MessageKind, Service, SubscriptionPlan, +}; async fn spawn_server() -> (String, Arc>>) { let handled = Arc::new(Mutex::new(Vec::::new())); @@ -17,20 +19,22 @@ async fn spawn_server() -> (String, Arc>>) { let service = Arc::new( Service::new(()) .event("order.created") - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { h.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({"ok": true})) + async move { Ok(json!({"ok": true})) } }) .event("flaky") - .handle(|_| { + .handle(|_ctx: &Context<()>| async move { Err(HandlerError::Repository( sourced_rust::RepositoryError::Model("transient".into()), )) }) .event("bad") - .handle(|_| Err(HandlerError::Rejected("permanent".into()))), + .handle( + |_ctx: &Context<()>| async move { Err(HandlerError::Rejected("permanent".into())) }, + ), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/tests/microsvc/basic.rs b/tests/microsvc/basic.rs index 141dcbb0b..cf0d66d14 100644 --- a/tests/microsvc/basic.rs +++ b/tests/microsvc/basic.rs @@ -1,49 +1,61 @@ //! Basic microsvc integration tests — exercises dispatch with a real repository. use serde_json::json; -use sourced_rust::microsvc::{HandlerError, Service, Session}; -use sourced_rust::{AggregateBuilder, HashMapRepository}; +use sourced_rust::microsvc::{Context, HandlerError, Service, Session}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository}; use crate::models::counter::{Counter, CreateCounter, DecrementCounter, IncrementCounter}; -#[test] -fn full_lifecycle() { +#[tokio::test] +async fn full_lifecycle() { let service = Service::with_repo(HashMapRepository::new()) .command("counter.create") - .handle(|ctx| { - let input = ctx.input::()?; - let counter_repo = ctx.repo().clone().aggregate::(); - let mut counter = Counter::default(); - counter.create(input.id.clone())?; - counter_repo.commit(&mut counter)?; - Ok(json!({ "id": input.id })) + .handle(|ctx: &Context| { + let input = ctx.input::(); + let counter_repo = ctx.repo().clone().async_aggregate::(); + async move { + let input = input?; + let mut counter = Counter::default(); + counter.create(input.id.clone())?; + counter_repo.commit(&mut counter).await?; + Ok(json!({ "id": input.id })) + } }) .command("counter.increment") - .handle(|ctx| { - let input = ctx.input::()?; - let counter_repo = ctx.repo().clone().aggregate::(); - let mut counter: Counter = counter_repo - .get(&input.id)? - .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; - counter.increment(input.amount)?; - counter_repo.commit(&mut counter)?; - Ok(json!({ "value": counter.value })) + .handle(|ctx: &Context| { + let input = ctx.input::(); + let counter_repo = ctx.repo().clone().async_aggregate::(); + async move { + let input = input?; + let mut counter: Counter = counter_repo + .get(&input.id) + .await? + .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; + counter.increment(input.amount)?; + counter_repo.commit(&mut counter).await?; + Ok(json!({ "value": counter.value })) + } }) .command("counter.decrement") - .handle(|ctx| { - let input = ctx.input::()?; - let counter_repo = ctx.repo().clone().aggregate::(); - let mut counter: Counter = counter_repo - .get(&input.id)? - .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; - counter.decrement(input.amount)?; - counter_repo.commit(&mut counter)?; - Ok(json!({ "value": counter.value })) + .handle(|ctx: &Context| { + let input = ctx.input::(); + let counter_repo = ctx.repo().clone().async_aggregate::(); + async move { + let input = input?; + let mut counter: Counter = counter_repo + .get(&input.id) + .await? + .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; + counter.decrement(input.amount)?; + counter_repo.commit(&mut counter).await?; + Ok(json!({ "value": counter.value })) + } }); // Create let result = service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); assert_eq!(result, json!({ "id": "c1" })); @@ -54,6 +66,7 @@ fn full_lifecycle() { json!({ "id": "c1", "amount": 5 }), Session::new(), ) + .await .unwrap(); assert_eq!(result, json!({ "value": 5 })); @@ -63,6 +76,7 @@ fn full_lifecycle() { json!({ "id": "c1", "amount": 3 }), Session::new(), ) + .await .unwrap(); // Decrement @@ -72,11 +86,12 @@ fn full_lifecycle() { json!({ "id": "c1", "amount": 2 }), Session::new(), ) + .await .unwrap(); assert_eq!(result, json!({ "value": 6 })); // Verify final state via repo - let counter_repo = service.repo().clone().aggregate::(); - let counter: Counter = counter_repo.get("c1").unwrap().unwrap(); + let counter_repo = service.repo().clone().async_aggregate::(); + let counter: Counter = counter_repo.get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 6); } diff --git a/tests/microsvc/convention.rs b/tests/microsvc/convention.rs index 236abce41..110f9bb35 100644 --- a/tests/microsvc/convention.rs +++ b/tests/microsvc/convention.rs @@ -9,7 +9,7 @@ use serde_json::json; use sourced_rust::microsvc::{Service, Session}; -use sourced_rust::{AggregateBuilder, HashMapRepository, OutboxStore, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, OutboxStore, Queueable}; use crate::handlers; use crate::models::counter::Counter; @@ -18,10 +18,10 @@ use crate::models::counter::Counter; // Handler convention — register, dispatch, verify // ============================================================================ -#[test] -fn register_handlers_and_dispatch() { +#[tokio::test] +async fn register_handlers_and_dispatch() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, command handlers::counter_increment, ); @@ -33,6 +33,7 @@ fn register_handlers_and_dispatch() { // Create let result = service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); assert_eq!(result, json!({ "id": "c1" })); @@ -43,37 +44,43 @@ fn register_handlers_and_dispatch() { json!({ "id": "c1", "amount": 10 }), Session::new(), ) + .await .unwrap(); assert_eq!(result, json!({ "id": "c1", "value": 10 })); // Verify state via repo - let counter: Counter = service.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 10); } -#[test] -fn guard_rejects_bad_input() { +#[tokio::test] +async fn guard_rejects_bad_input() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, ); - let result = service.dispatch("counter.create", json!({ "wrong": 1 }), Session::new()); + let result = service + .dispatch("counter.create", json!({ "wrong": 1 }), Session::new()) + .await; assert!(result.is_err()); } -#[test] -fn handler_rejects_duplicate_create() { +#[tokio::test] +async fn handler_rejects_duplicate_create() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, ); service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); - let result = service.dispatch("counter.create", json!({ "id": "c1" }), Session::new()); + let result = service + .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await; assert!(result.is_err()); } @@ -81,22 +88,23 @@ fn handler_rejects_duplicate_create() { // Outbox — handlers commit aggregate + outbox message atomically // ============================================================================ -#[test] -fn create_persists_outbox_message() { +#[tokio::test] +async fn create_persists_outbox_message() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, ); let result = service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); assert_eq!(result, json!({ "id": "c1" })); let inner = service.repo().repo().inner(); // Aggregate was persisted - let counter: Counter = service.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 0); // Outbox message was persisted @@ -105,19 +113,22 @@ fn create_persists_outbox_message() { assert_eq!(pending[0].event_type, "CounterCreated"); } -#[test] -fn duplicate_create_leaves_single_outbox_message() { +#[tokio::test] +async fn duplicate_create_leaves_single_outbox_message() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, ); service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); // Second create fails — no duplicate outbox message - let result = service.dispatch("counter.create", json!({ "id": "c1" }), Session::new()); + let result = service + .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await; assert!(result.is_err()); let pending = service @@ -130,16 +141,17 @@ fn duplicate_create_leaves_single_outbox_message() { assert_eq!(pending.len(), 1); } -#[test] -fn increment_persists_outbox_message() { +#[tokio::test] +async fn increment_persists_outbox_message() { let service = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, command handlers::counter_increment, ); service .dispatch("counter.create", json!({ "id": "c1" }), Session::new()) + .await .unwrap(); service @@ -148,10 +160,11 @@ fn increment_persists_outbox_message() { json!({ "id": "c1", "amount": 7 }), Session::new(), ) + .await .unwrap(); // Aggregate state is correct - let counter: Counter = service.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 7); // Both outbox messages were persisted diff --git a/tests/microsvc/handlers/counter_create.rs b/tests/microsvc/handlers/counter_create.rs index 13595134c..f0b24fbcc 100644 --- a/tests/microsvc/handlers/counter_create.rs +++ b/tests/microsvc/handlers/counter_create.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use super::Repo; use crate::models::counter::Counter; @@ -24,10 +24,10 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; - if ctx.repo().get(&input.id)?.is_some() { + if ctx.repo().get(&input.id).await?.is_some() { return Err(HandlerError::Rejected(format!( "counter {} already exists", input.id @@ -39,7 +39,7 @@ pub fn handle(ctx: &Context) -> Result { let message = OutboxMessage::domain_event("CounterCreated", &counter)?; - ctx.repo().outbox_sync(message).commit_sync(&mut counter)?; + ctx.repo().outbox(message).commit(&mut counter).await?; Ok(json!({ "id": input.id })) } diff --git a/tests/microsvc/handlers/counter_increment.rs b/tests/microsvc/handlers/counter_increment.rs index 90fff7b62..268394624 100644 --- a/tests/microsvc/handlers/counter_increment.rs +++ b/tests/microsvc/handlers/counter_increment.rs @@ -3,7 +3,7 @@ use serde::Deserialize; use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +use sourced_rust::OutboxMessage; use super::Repo; use crate::models::counter::Counter; @@ -20,19 +20,20 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["id", "amount"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut counter: Counter = ctx .repo() - .get(&input.id)? + .get(&input.id) + .await? .ok_or_else(|| HandlerError::NotFound(input.id.clone()))?; counter.increment(input.amount)?; let message = OutboxMessage::domain_event("CounterIncremented", &counter)?; - ctx.repo().outbox_sync(message).commit_sync(&mut counter)?; + ctx.repo().outbox(message).commit(&mut counter).await?; Ok(json!({ "id": input.id, "value": counter.value })) } diff --git a/tests/microsvc/handlers/mod.rs b/tests/microsvc/handlers/mod.rs index 10bade81a..2dbc20fa0 100644 --- a/tests/microsvc/handlers/mod.rs +++ b/tests/microsvc/handlers/mod.rs @@ -1,8 +1,13 @@ -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; use crate::models::counter::Counter; -pub type Repo = AggregateRepository, Counter>; +pub type Repo = AsyncAggregateRepository< + QueuedRepository, + Counter, +>; pub mod counter_create; pub mod counter_increment; diff --git a/tests/microsvc/handlers/whoami.rs b/tests/microsvc/handlers/whoami.rs index e811c481d..09d49ab64 100644 --- a/tests/microsvc/handlers/whoami.rs +++ b/tests/microsvc/handlers/whoami.rs @@ -13,7 +13,7 @@ pub fn guard(_ctx: &Context) -> bool { true } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let user_id = ctx.user_id()?; Ok(json!({ "user_id": user_id })) } diff --git a/tests/microsvc/session.rs b/tests/microsvc/session.rs index d48ea188a..f3f549d01 100644 --- a/tests/microsvc/session.rs +++ b/tests/microsvc/session.rs @@ -1,31 +1,44 @@ //! Session integration tests — exercises session variables through dispatch. use serde_json::json; -use sourced_rust::microsvc::{HandlerError, Service, Session}; +use sourced_rust::microsvc::{Context, HandlerError, Service, Session}; use std::collections::HashMap; -#[test] -fn handler_accesses_user_id() { - let service = Service::new(()).command("whoami").handle(|ctx| { - let user_id = ctx.user_id()?; - Ok(json!({ "user_id": user_id })) - }); +#[tokio::test] +async fn handler_accesses_user_id() { + let service = Service::new(()) + .command("whoami") + .handle(|ctx: &Context<()>| { + let user_id = ctx.user_id().map(|id| id.to_string()); + async move { + let user_id = user_id?; + Ok(json!({ "user_id": user_id })) + } + }); let mut vars = HashMap::new(); vars.insert("x-hasura-user-id".to_string(), "user-42".to_string()); let session = Session::from_map(vars); - let result = service.dispatch("whoami", json!({}), session).unwrap(); + let result = service + .dispatch("whoami", json!({}), session) + .await + .unwrap(); assert_eq!(result, json!({ "user_id": "user-42" })); } -#[test] -fn missing_user_id_returns_unauthorized() { - let service = Service::new(()).command("whoami").handle(|ctx| { - let _user_id = ctx.user_id()?; - Ok(json!({})) - }); +#[tokio::test] +async fn missing_user_id_returns_unauthorized() { + let service = Service::new(()) + .command("whoami") + .handle(|ctx: &Context<()>| { + let user_id = ctx.user_id().map(|id| id.to_string()); + async move { + let _user_id = user_id?; + Ok(json!({})) + } + }); - let result = service.dispatch("whoami", json!({}), Session::new()); + let result = service.dispatch("whoami", json!({}), Session::new()).await; assert!(matches!(result, Err(HandlerError::Unauthorized(_)))); } diff --git a/tests/microsvc/transport_grpc.rs b/tests/microsvc/transport_grpc.rs index 525cb9c6d..cfe762cd8 100644 --- a/tests/microsvc/transport_grpc.rs +++ b/tests/microsvc/transport_grpc.rs @@ -9,7 +9,7 @@ use sourced_rust::microsvc::grpc::{ CommandServiceClient, GrpcRequest, GrpcServeError, HealthRequest, }; use sourced_rust::microsvc::Service; -use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, Queueable}; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; @@ -19,7 +19,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, diff --git a/tests/microsvc/transport_http.rs b/tests/microsvc/transport_http.rs index 1466f07e2..bf8d7a351 100644 --- a/tests/microsvc/transport_http.rs +++ b/tests/microsvc/transport_http.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use serde_json::json; use sourced_rust::microsvc::{self, Service}; -use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, Queueable}; use crate::handlers; use crate::handlers::Repo; @@ -14,7 +14,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, diff --git a/tests/microsvc/transport_listen.rs b/tests/microsvc/transport_listen.rs index bb6dfdefd..6227eb610 100644 --- a/tests/microsvc/transport_listen.rs +++ b/tests/microsvc/transport_listen.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use serde_json::json; use sourced_rust::microsvc::transport::{Bus, BusConsumer, FailurePolicy, InMemoryBus, RunOptions}; use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; -use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, Queueable}; use crate::handlers; use crate::handlers::Repo; @@ -20,7 +20,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, @@ -52,7 +52,7 @@ async fn dispatches_from_queue() { .await .expect("listen should drain the command queues"); - let counter: Counter = service.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 10); } @@ -91,7 +91,7 @@ async fn tolerates_handler_failures_and_keeps_processing() { // The good aggregate was still created and incremented, proving the failure // did not stop the consumer. - let c2: Counter = service.repo().get("c2").unwrap().unwrap(); + let c2: Counter = service.repo().get("c2").await.unwrap().unwrap(); assert_eq!(c2.value, 7); } @@ -111,10 +111,11 @@ async fn coexists_with_direct_dispatch() { // c2 created via direct dispatch on the same service. service .dispatch("counter.create", json!({ "id": "c2" }), Session::new()) + .await .expect("direct dispatch should create c2"); - let c1: Counter = service.repo().get("c1").unwrap().unwrap(); - let c2: Counter = service.repo().get("c2").unwrap().unwrap(); + let c1: Counter = service.repo().get("c1").await.unwrap().unwrap(); + let c2: Counter = service.repo().get("c2").await.unwrap().unwrap(); assert_eq!(c1.value, 0); assert_eq!(c2.value, 0); } @@ -162,11 +163,11 @@ async fn multiple_services_on_different_queues() { let store = HashMapRepository::new(); let service_a = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(store.clone().queued().aggregate::()), + Service::with_repo(store.clone().queued_async().async_aggregate::()), command handlers::counter_create, )); let service_b = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(store.queued().aggregate::()), + Service::with_repo(store.queued_async().async_aggregate::()), command handlers::counter_increment, )); @@ -191,6 +192,6 @@ async fn multiple_services_on_different_queues() { .await .expect("service B should drain the increment queue"); - let counter: Counter = service_a.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service_a.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 42); } diff --git a/tests/microsvc/transport_subscribe.rs b/tests/microsvc/transport_subscribe.rs index 9f01e08e5..12a47c615 100644 --- a/tests/microsvc/transport_subscribe.rs +++ b/tests/microsvc/transport_subscribe.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; use sourced_rust::microsvc::{Message, MessageKind, Service}; -use sourced_rust::{AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository, Queueable}; use crate::handlers; use crate::handlers::Repo; @@ -15,17 +15,21 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new( - Service::with_repo(HashMapRepository::new().queued().aggregate::()) - .event(handlers::counter_create::COMMAND) - .guarded( - handlers::counter_create::guard, - handlers::counter_create::handle, - ) - .event(handlers::counter_increment::COMMAND) - .guarded( - handlers::counter_increment::guard, - handlers::counter_increment::handle, - ), + Service::with_repo( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ) + .event(handlers::counter_create::COMMAND) + .guarded( + handlers::counter_create::guard, + handlers::counter_create::handle, + ) + .event(handlers::counter_increment::COMMAND) + .guarded( + handlers::counter_increment::guard, + handlers::counter_increment::handle, + ), ) } @@ -59,6 +63,6 @@ async fn dispatches_from_pubsub() { .await .expect("subscriber should drain the bus"); - let counter: Counter = service.repo().get("c1").unwrap().unwrap(); + let counter: Counter = service.repo().get("c1").await.unwrap().unwrap(); assert_eq!(counter.value, 15); } diff --git a/tests/nats_transport/main.rs b/tests/nats_transport/main.rs index 37cc88876..316ccfedb 100644 --- a/tests/nats_transport/main.rs +++ b/tests/nats_transport/main.rs @@ -13,7 +13,7 @@ use sourced_rust::microsvc::transport::{ run_source, AsyncMessagePublisher, Bus, BusConsumer, NatsBus, NatsJetStreamSource, NatsPublisher, RunOptions, }; -use sourced_rust::microsvc::{Message, MessageKind, Service}; +use sourced_rust::microsvc::{Context, Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); @@ -80,12 +80,12 @@ async fn publish_then_consume_round_trips_through_jetstream() { let service = Arc::new( Service::new(()) .event(Box::leak(subject.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { assert_eq!(ctx.message().name(), subject_for_handler); h.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } }), ); @@ -126,14 +126,15 @@ async fn message_id_and_metadata_survive_the_round_trip() { let service = Arc::new( Service::new(()) .event(Box::leak(subject.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { let m = ctx.message(); - *o.lock().unwrap() = Some(( + let recorded = Some(( m.id().map(str::to_string), m.correlation_id().map(str::to_string), m.payload().to_vec(), )); - Ok(json!({})) + *o.lock().unwrap() = recorded; + async move { Ok(json!({})) } }), ); run_source(service, source, RunOptions::idempotent()) @@ -159,11 +160,11 @@ fn recording_service( MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), }; - Arc::new(registered.handle(move |ctx| { + Arc::new(registered.handle(move |ctx: &Context<()>| { rec.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } })) } diff --git a/tests/postgres_transport/main.rs b/tests/postgres_transport/main.rs index 7f084e1ba..70fb72fb7 100644 --- a/tests/postgres_transport/main.rs +++ b/tests/postgres_transport/main.rs @@ -16,7 +16,7 @@ use sourced_rust::microsvc::transport::{ run_source, AsyncMessageSource, Bus, BusConsumer, OutboxSource, PostgresBus, ReceivedMessage, RunOptions, }; -use sourced_rust::microsvc::{Message, MessageKind, Service}; +use sourced_rust::microsvc::{Context, Message, MessageKind, Service}; use sourced_rust::{ AsyncCommitBatch, AsyncOutboxStore, AsyncTransactionalCommit, OutboxMessage, OutboxMessageStatus, PostgresOutboxStore, PostgresRepository, @@ -55,13 +55,17 @@ async fn status(store: &PostgresOutboxStore, id: &str) -> Option>>) -> Arc> { - Arc::new(Service::new(()).event("evt").handle(move |ctx| { - handled - .lock() - .unwrap() - .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) - })) + Arc::new( + Service::new(()) + .event("evt") + .handle(move |ctx: &Context<()>| { + handled + .lock() + .unwrap() + .push(ctx.message().id().unwrap_or_default().to_string()); + async move { Ok(json!({})) } + }), + ) } #[tokio::test] @@ -196,11 +200,11 @@ fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), }; - Arc::new(registered.handle(move |ctx| { + Arc::new(registered.handle(move |ctx: &Context<()>| { rec.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } })) } diff --git a/tests/rabbitmq_transport/main.rs b/tests/rabbitmq_transport/main.rs index 034bff57e..73eed0910 100644 --- a/tests/rabbitmq_transport/main.rs +++ b/tests/rabbitmq_transport/main.rs @@ -12,7 +12,7 @@ use sourced_rust::microsvc::transport::{ run_source, AsyncMessagePublisher, Bus, BusConsumer, RabbitBus, RabbitPublisher, RabbitSource, RunOptions, }; -use sourced_rust::microsvc::{Message, MessageKind, Service}; +use sourced_rust::microsvc::{Context, Message, MessageKind, Service}; static SEQ: AtomicU64 = AtomicU64::new(1); @@ -54,11 +54,11 @@ fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), }; - Arc::new(registered.handle(move |ctx| { + Arc::new(registered.handle(move |ctx: &Context<()>| { rec.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } })) } @@ -86,11 +86,11 @@ async fn publish_then_consume_round_trips_through_rabbitmq() { let service = Arc::new( Service::new(()) .event(Box::leak(queue.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { h.lock() .unwrap() .push(ctx.message().id().unwrap_or_default().to_string()); - Ok(json!({})) + async move { Ok(json!({})) } }), ); run_source(service, source, RunOptions::idempotent()) @@ -129,15 +129,16 @@ async fn message_id_and_metadata_survive_the_round_trip() { let service = Arc::new( Service::new(()) .event(Box::leak(queue.clone().into_boxed_str())) - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { let m = ctx.message(); - *o.lock().unwrap() = Some(( + let recorded = Some(( m.id().map(str::to_string), m.correlation_id().map(str::to_string), m.payload().to_vec(), m.content_type.clone(), )); - Ok(json!({})) + *o.lock().unwrap() = recorded; + async move { Ok(json!({})) } }), ); run_source(service, source, RunOptions::idempotent()) diff --git a/tests/read_model_session/main.rs b/tests/read_model_session/main.rs index 696f6796d..c1d686ddd 100644 --- a/tests/read_model_session/main.rs +++ b/tests/read_model_session/main.rs @@ -2,9 +2,9 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use sourced_rust::{ - ExpectedVersion, InMemoryReadModelStore, PatchMode, ReadModel, ReadModelAdapterCapabilities, - ReadModelError, ReadModelMutation, ReadModelWorkspaceExt, ReadModelWritePlanBuilder, RowKey, - RowPatch, RowValue, RowWriteMode, Versioned, + AsyncReadModelWorkspaceExt, ExpectedVersion, InMemoryReadModelStore, PatchMode, ReadModel, + ReadModelAdapterCapabilities, ReadModelError, ReadModelMutation, ReadModelWritePlanBuilder, + RowKey, RowPatch, RowValue, RowWriteMode, Versioned, }; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ReadModel)] @@ -171,8 +171,8 @@ fn insert_and_upsert_patch_carry_explicit_missing_row_behavior() { assert_eq!(upsert_patch.mode, PatchMode::InsertMissing); } -#[test] -fn insert_missing_patch_builds_full_row_from_key_before_insert() { +#[tokio::test] +async fn insert_missing_patch_builds_full_row_from_key_before_insert() { let store = InMemoryReadModelStore::new(); store.register_schema::().unwrap(); let patch = RowPatch::new() @@ -189,12 +189,13 @@ fn insert_missing_patch_builds_full_row_from_key_before_insert() { session .upsert_patch::(account_key("acct-1"), patch) .unwrap(); - session.commit(&store).unwrap(); + session.commit_async(&store).await.unwrap(); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); let loaded = read_models - .load::(account_key("acct-1")) + .load_async::(account_key("acct-1")) .one() + .await .unwrap() .unwrap(); assert_eq!(loaded.data.account_id, "acct-1"); @@ -203,8 +204,8 @@ fn insert_missing_patch_builds_full_row_from_key_before_insert() { assert_eq!(loaded.data.deposit_count, 2); } -#[test] -fn insert_missing_patch_rejects_primary_key_mismatch() { +#[tokio::test] +async fn insert_missing_patch_rejects_primary_key_mismatch() { let store = InMemoryReadModelStore::new(); let patch = RowPatch::new() .set("account_id", RowValue::String("acct-2".into())) @@ -217,15 +218,15 @@ fn insert_missing_patch_rejects_primary_key_mismatch() { session .upsert_patch::(account_key("acct-1"), patch) .unwrap(); - let err = session.commit(&store).unwrap_err(); + let err = session.commit_async(&store).await.unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("primary-key column `account_id`")) ); } -#[test] -fn insert_missing_patch_rejects_partial_new_row() { +#[tokio::test] +async fn insert_missing_patch_rejects_partial_new_row() { let store = InMemoryReadModelStore::new(); store.register_schema::().unwrap(); let patch = RowPatch::new().set("owner", RowValue::String("Grace".into())); @@ -234,26 +235,27 @@ fn insert_missing_patch_rejects_partial_new_row() { session .upsert_patch::(account_key("acct-1"), patch) .unwrap(); - let err = session.commit(&store).unwrap_err(); + let err = session.commit_async(&store).await.unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("missing required column `balance_cents`")) ); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); let loaded = read_models - .load::(account_key("acct-1")) + .load_async::(account_key("acct-1")) .one() + .await .unwrap(); assert!(loaded.is_none()); } -#[test] -fn existing_patch_rejects_primary_key_mismatch() { +#[tokio::test] +async fn existing_patch_rejects_primary_key_mismatch() { let store = InMemoryReadModelStore::new(); let mut setup = ReadModelWritePlanBuilder::new(); setup.upsert(&AccountSummary::new("acct-1")).unwrap(); - setup.commit(&store).unwrap(); + setup.commit_async(&store).await.unwrap(); let patch = RowPatch::new() .set("account_id", RowValue::String("acct-2".into())) .set("owner", RowValue::String("Grace".into())); @@ -262,7 +264,7 @@ fn existing_patch_rejects_primary_key_mismatch() { session .patch::(account_key("acct-1"), patch) .unwrap(); - let err = session.commit(&store).unwrap_err(); + let err = session.commit_async(&store).await.unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("primary-key column `account_id`")) diff --git a/tests/sagas/handlers/inventory/init.rs b/tests/sagas/handlers/inventory/init.rs index 7f3279470..0b845b1a4 100644 --- a/tests/sagas/handlers/inventory/init.rs +++ b/tests/sagas/handlers/inventory/init.rs @@ -6,12 +6,12 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["sku", "stock"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut inv = Inventory::new(); inv.initialize(input.sku.clone(), input.stock)?; - ctx.repo().commit(&mut inv)?; + ctx.repo().commit(&mut inv).await?; Ok(json!({ "sku": input.sku, "stock": input.stock })) } diff --git a/tests/sagas/handlers/inventory/mod.rs b/tests/sagas/handlers/inventory/mod.rs index 97c07b939..242a46890 100644 --- a/tests/sagas/handlers/inventory/mod.rs +++ b/tests/sagas/handlers/inventory/mod.rs @@ -2,12 +2,17 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository, SyncOutboxCommitExt}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; use super::messages::*; use crate::order::Inventory; -pub type Repo = AggregateRepository, Inventory>; +pub type Repo = AsyncAggregateRepository< + QueuedRepository, + Inventory, +>; pub mod init; pub mod reserve; diff --git a/tests/sagas/handlers/inventory/reserve.rs b/tests/sagas/handlers/inventory/reserve.rs index b717279e0..e5128bea2 100644 --- a/tests/sagas/handlers/inventory/reserve.rs +++ b/tests/sagas/handlers/inventory/reserve.rs @@ -6,12 +6,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id", "sku", "quantity"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut inv = ctx .repo() - .get(&input.sku)? + .get(&input.sku) + .await? .ok_or_else(|| HandlerError::NotFound(input.sku.clone()))?; if !inv.can_reserve(input.quantity) { @@ -29,6 +30,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut inv)?; + ctx.repo().outbox(msg).commit(&mut inv).await?; Ok(json!({ "reserved": input.quantity })) } diff --git a/tests/sagas/handlers/orders/complete.rs b/tests/sagas/handlers/orders/complete.rs index f7b090fc5..5ca8b1657 100644 --- a/tests/sagas/handlers/orders/complete.rs +++ b/tests/sagas/handlers/orders/complete.rs @@ -6,12 +6,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut order = ctx .repo() - .get(&input.order_id)? + .get(&input.order_id) + .await? .ok_or_else(|| HandlerError::NotFound(input.order_id.clone()))?; order.mark_inventory_reserved()?; order.mark_payment_processed()?; @@ -27,6 +28,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut order)?; + ctx.repo().outbox(msg).commit(&mut order).await?; Ok(json!({ "order_id": input.order_id })) } diff --git a/tests/sagas/handlers/orders/create.rs b/tests/sagas/handlers/orders/create.rs index 151fca224..8b33613d3 100644 --- a/tests/sagas/handlers/orders/create.rs +++ b/tests/sagas/handlers/orders/create.rs @@ -6,7 +6,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id", "customer_id", "items"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut order = Order::new(); @@ -22,6 +22,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut order)?; + ctx.repo().outbox(msg).commit(&mut order).await?; Ok(json!({ "order_id": input.order_id })) } diff --git a/tests/sagas/handlers/orders/mod.rs b/tests/sagas/handlers/orders/mod.rs index e573efacf..5b1423af1 100644 --- a/tests/sagas/handlers/orders/mod.rs +++ b/tests/sagas/handlers/orders/mod.rs @@ -2,12 +2,15 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository, SyncOutboxCommitExt}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; use super::messages::*; use crate::order::Order; -pub type Repo = AggregateRepository, Order>; +pub type Repo = + AsyncAggregateRepository, Order>; pub mod complete; pub mod create; diff --git a/tests/sagas/handlers/payments/mod.rs b/tests/sagas/handlers/payments/mod.rs index d0b80ae65..49e2c66d9 100644 --- a/tests/sagas/handlers/payments/mod.rs +++ b/tests/sagas/handlers/payments/mod.rs @@ -2,11 +2,16 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository, SyncOutboxCommitExt}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; use super::messages::*; use crate::order::Payment; -pub type Repo = AggregateRepository, Payment>; +pub type Repo = AsyncAggregateRepository< + QueuedRepository, + Payment, +>; pub mod process; diff --git a/tests/sagas/handlers/payments/process.rs b/tests/sagas/handlers/payments/process.rs index 953213c35..0d83a2f21 100644 --- a/tests/sagas/handlers/payments/process.rs +++ b/tests/sagas/handlers/payments/process.rs @@ -6,7 +6,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id", "amount_cents"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let payment_id = format!("pay-{}", input.order_id); @@ -29,6 +29,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut payment)?; + ctx.repo().outbox(msg).commit(&mut payment).await?; Ok(json!({ "payment_id": payment_id })) } diff --git a/tests/sagas/handlers/saga/mod.rs b/tests/sagas/handlers/saga/mod.rs index 7ca5a7052..479dbd431 100644 --- a/tests/sagas/handlers/saga/mod.rs +++ b/tests/sagas/handlers/saga/mod.rs @@ -2,12 +2,17 @@ use serde_json::{json, Value}; use sourced_rust::microsvc::{Context, HandlerError}; -use sourced_rust::{AggregateRepository, HashMapRepository, QueuedRepository, SyncOutboxCommitExt}; +use sourced_rust::{ + AsyncAggregateRepository, HashMapRepository, InMemoryAsyncLockManager, QueuedRepository, +}; use super::messages::*; use crate::order::OrderFulfillmentSaga; -pub type Repo = AggregateRepository, OrderFulfillmentSaga>; +pub type Repo = AsyncAggregateRepository< + QueuedRepository, + OrderFulfillmentSaga, +>; pub mod on_inventory_reserved; pub mod on_order_completed; diff --git a/tests/sagas/handlers/saga/on_inventory_reserved.rs b/tests/sagas/handlers/saga/on_inventory_reserved.rs index ec20a5ecb..0763926b1 100644 --- a/tests/sagas/handlers/saga/on_inventory_reserved.rs +++ b/tests/sagas/handlers/saga/on_inventory_reserved.rs @@ -6,12 +6,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut saga = ctx .repo() - .get(&input.saga_id)? + .get(&input.saga_id) + .await? .ok_or_else(|| HandlerError::NotFound(input.saga_id.clone()))?; saga.inventory_reserved()?; @@ -26,6 +27,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut saga)?; + ctx.repo().outbox(msg).commit(&mut saga).await?; Ok(json!({ "next": "ProcessPayment" })) } diff --git a/tests/sagas/handlers/saga/on_order_completed.rs b/tests/sagas/handlers/saga/on_order_completed.rs index 02158e1f9..f0a46d6d7 100644 --- a/tests/sagas/handlers/saga/on_order_completed.rs +++ b/tests/sagas/handlers/saga/on_order_completed.rs @@ -6,15 +6,16 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut saga = ctx .repo() - .get(&input.saga_id)? + .get(&input.saga_id) + .await? .ok_or_else(|| HandlerError::NotFound(input.saga_id.clone()))?; saga.complete()?; - ctx.repo().commit(&mut saga)?; + ctx.repo().commit(&mut saga).await?; Ok(json!({ "saga_id": input.saga_id, "status": "completed" })) } diff --git a/tests/sagas/handlers/saga/on_order_created.rs b/tests/sagas/handlers/saga/on_order_created.rs index 3cec0f0c6..6a7c773bf 100644 --- a/tests/sagas/handlers/saga/on_order_created.rs +++ b/tests/sagas/handlers/saga/on_order_created.rs @@ -6,12 +6,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut saga = ctx .repo() - .get(&input.saga_id)? + .get(&input.saga_id) + .await? .ok_or_else(|| HandlerError::NotFound(input.saga_id.clone()))?; let sku = saga.items()[0].sku.clone(); @@ -29,6 +30,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut saga)?; + ctx.repo().outbox(msg).commit(&mut saga).await?; Ok(json!({ "next": "ReserveInventory" })) } diff --git a/tests/sagas/handlers/saga/on_payment_succeeded.rs b/tests/sagas/handlers/saga/on_payment_succeeded.rs index 31df343f7..358e63664 100644 --- a/tests/sagas/handlers/saga/on_payment_succeeded.rs +++ b/tests/sagas/handlers/saga/on_payment_succeeded.rs @@ -6,12 +6,13 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut saga = ctx .repo() - .get(&input.saga_id)? + .get(&input.saga_id) + .await? .ok_or_else(|| HandlerError::NotFound(input.saga_id.clone()))?; saga.payment_succeeded()?; @@ -25,6 +26,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut saga)?; + ctx.repo().outbox(msg).commit(&mut saga).await?; Ok(json!({ "next": "CompleteOrder" })) } diff --git a/tests/sagas/handlers/saga/start.rs b/tests/sagas/handlers/saga/start.rs index cf66cb168..b654092a6 100644 --- a/tests/sagas/handlers/saga/start.rs +++ b/tests/sagas/handlers/saga/start.rs @@ -6,7 +6,7 @@ pub fn guard(ctx: &Context) -> bool { ctx.has_fields(&["saga_id", "order_id", "customer_id", "items", "total_cents"]) } -pub fn handle(ctx: &Context) -> Result { +pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let input = ctx.input::()?; let mut saga = OrderFulfillmentSaga::new(); @@ -31,6 +31,6 @@ pub fn handle(ctx: &Context) -> Result { }, )?; - ctx.repo().outbox_sync(msg).commit_sync(&mut saga)?; + ctx.repo().outbox(msg).commit(&mut saga).await?; Ok(json!({ "saga_id": input.saga_id })) } diff --git a/tests/sagas/microsvc_saga.rs b/tests/sagas/microsvc_saga.rs index dabd5630f..b2f24cb57 100644 --- a/tests/sagas/microsvc_saga.rs +++ b/tests/sagas/microsvc_saga.rs @@ -4,7 +4,7 @@ //! organized by service domain under `handlers/`. //! //! Each service is typed to a specific aggregate via -//! `Service::with_repo(repo.queued().aggregate::())`, so handlers access +//! `Service::with_repo(repo.queued_async().async_aggregate::())`, so handlers access //! `ctx.repo().get()`, `ctx.repo().commit()`, etc. directly. //! //! Two tests: @@ -19,8 +19,8 @@ use serde_json::json; use sourced_rust::microsvc::transport::{Bus, BusConsumer, InMemoryBus, RunOptions}; use sourced_rust::microsvc::{Message, MessageKind, Service, Session}; use sourced_rust::{ - AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, - OutboxClaimRef, Queueable, + AsyncAggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, + HashMapRepository, OutboxClaimRef, Queueable, }; use super::handlers; @@ -51,13 +51,13 @@ fn event_message(name: &str, input: serde_json::Value) -> Message { /// │ dispatch ──→ [Payment Service] typed: Payment │ /// └────────────────────────────────────────────────────────────────┘ /// ``` -#[test] -fn saga_orchestrated() { +#[tokio::test] +async fn saga_orchestrated() { let saga_svc = sourced_rust::register_handlers!( Service::with_repo( HashMapRepository::new() - .queued() - .aggregate::() + .queued_async() + .async_aggregate::() ), command handlers::saga::start, event handlers::saga::on_order_created, @@ -67,19 +67,19 @@ fn saga_orchestrated() { ); let order_svc = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::orders::create, command handlers::orders::complete, ); let inventory_svc = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::inventory::init, command handlers::inventory::reserve, ); let payment_svc = sourced_rust::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::with_repo(HashMapRepository::new().queued_async().async_aggregate::()), command handlers::payments::process, ); @@ -92,6 +92,7 @@ fn saga_orchestrated() { json!({ "sku": "WIDGET-001", "stock": 100 }), s(), ) + .await .unwrap(); // 2. Start saga → creates saga + outbox(CreateOrder) @@ -107,6 +108,7 @@ fn saga_orchestrated() { }), s(), ) + .await .unwrap(); // 3. Create order → outbox(OrderCreated) @@ -122,6 +124,7 @@ fn saga_orchestrated() { }), s(), ) + .await .unwrap(); // 4. Saga: order created → outbox(ReserveInventory) @@ -130,6 +133,7 @@ fn saga_orchestrated() { "OrderCreated", json!({ "saga_id": "saga-001", "order_id": "order-001" }), )) + .await .unwrap(); // 5. Reserve inventory → outbox(InventoryReserved) @@ -144,6 +148,7 @@ fn saga_orchestrated() { }), s(), ) + .await .unwrap(); // 6. Saga: inventory reserved → outbox(ProcessPayment) @@ -152,6 +157,7 @@ fn saga_orchestrated() { "InventoryReserved", json!({ "saga_id": "saga-001", "order_id": "order-001" }), )) + .await .unwrap(); // 7. Process payment → outbox(PaymentSucceeded) @@ -165,6 +171,7 @@ fn saga_orchestrated() { }), s(), ) + .await .unwrap(); // 8. Saga: payment succeeded → outbox(CompleteOrder) @@ -173,6 +180,7 @@ fn saga_orchestrated() { "PaymentSucceeded", json!({ "saga_id": "saga-001", "order_id": "order-001" }), )) + .await .unwrap(); // 9. Complete order → outbox(OrderCompleted) @@ -182,6 +190,7 @@ fn saga_orchestrated() { json!({ "saga_id": "saga-001", "order_id": "order-001" }), s(), ) + .await .unwrap(); // 10. Saga: order completed → saga done @@ -190,21 +199,32 @@ fn saga_orchestrated() { "OrderCompleted", json!({ "saga_id": "saga-001", "order_id": "order-001" }), )) + .await .unwrap(); // === Verify final state — typed repos return aggregates directly === - let saga = saga_svc.repo().peek("saga-001").unwrap().unwrap(); + let saga = saga_svc.repo().peek("saga-001").await.unwrap().unwrap(); assert_eq!(saga.status(), SagaStatus::Completed); assert!(saga.is_complete()); - let order = order_svc.repo().peek("order-001").unwrap().unwrap(); + let order = order_svc.repo().peek("order-001").await.unwrap().unwrap(); assert_eq!(order.status(), OrderStatus::Completed); - let inv = inventory_svc.repo().peek("WIDGET-001").unwrap().unwrap(); + let inv = inventory_svc + .repo() + .peek("WIDGET-001") + .await + .unwrap() + .unwrap(); assert_eq!(inv.available(), 95); - let payment = payment_svc.repo().peek("pay-order-001").unwrap().unwrap(); + let payment = payment_svc + .repo() + .peek("pay-order-001") + .await + .unwrap() + .unwrap(); assert!(payment.is_successful()); } @@ -271,7 +291,7 @@ async fn saga_distributed() { let saga_repo = HashMapRepository::new(); let saga_outbox = saga_repo.outbox_store(); let saga_svc = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(saga_repo.queued().aggregate::()), + Service::with_repo(saga_repo.queued_async().async_aggregate::()), command handlers::saga::start, event handlers::saga::on_order_created, event handlers::saga::on_inventory_reserved, @@ -283,7 +303,7 @@ async fn saga_distributed() { let order_repo = HashMapRepository::new(); let order_outbox = order_repo.outbox_store(); let order_svc = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(order_repo.queued().aggregate::()), + Service::with_repo(order_repo.queued_async().async_aggregate::()), command handlers::orders::create, command handlers::orders::complete, )); @@ -292,13 +312,13 @@ async fn saga_distributed() { let inventory_repo = HashMapRepository::new(); let inventory_outbox = inventory_repo.outbox_store(); { - let tmp = inventory_repo.clone().aggregate::(); + let tmp = inventory_repo.clone().async_aggregate::(); let mut inv = Inventory::new(); inv.initialize("WIDGET-001".to_string(), 100).unwrap(); - tmp.commit(&mut inv).unwrap(); + tmp.commit(&mut inv).await.unwrap(); } let inventory_svc = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(inventory_repo.queued().aggregate::()), + Service::with_repo(inventory_repo.queued_async().async_aggregate::()), command handlers::inventory::init, command handlers::inventory::reserve, )); @@ -307,7 +327,7 @@ async fn saga_distributed() { let payment_repo = HashMapRepository::new(); let payment_outbox = payment_repo.outbox_store(); let payment_svc = Arc::new(sourced_rust::register_handlers!( - Service::with_repo(payment_repo.queued().aggregate::()), + Service::with_repo(payment_repo.queued_async().async_aggregate::()), command handlers::payments::process, )); @@ -324,6 +344,7 @@ async fn saga_distributed() { }), Session::new(), ) + .await .unwrap(); // === DRIVE THE CHOREOGRAPHY OVER THE BUS UNTIL QUIESCENT === @@ -362,17 +383,27 @@ async fn saga_distributed() { // === VERIFY FINAL STATE — typed repos return aggregates directly === - let saga = saga_svc.repo().peek("saga-001").unwrap().unwrap(); + let saga = saga_svc.repo().peek("saga-001").await.unwrap().unwrap(); assert_eq!(saga.status(), SagaStatus::Completed); assert!(saga.is_complete()); - let order = order_svc.repo().peek("order-001").unwrap().unwrap(); + let order = order_svc.repo().peek("order-001").await.unwrap().unwrap(); assert_eq!(order.status(), OrderStatus::Completed); - let inv = inventory_svc.repo().peek("WIDGET-001").unwrap().unwrap(); + let inv = inventory_svc + .repo() + .peek("WIDGET-001") + .await + .unwrap() + .unwrap(); assert_eq!(inv.available(), 95); assert_eq!(inv.reserved(), 5); - let payment = payment_svc.repo().peek("pay-order-001").unwrap().unwrap(); + let payment = payment_svc + .repo() + .peek("pay-order-001") + .await + .unwrap() + .unwrap(); assert!(payment.is_successful()); } diff --git a/tests/sagas/orchestration.rs b/tests/sagas/orchestration.rs index ea48c9d63..e5643df76 100644 --- a/tests/sagas/orchestration.rs +++ b/tests/sagas/orchestration.rs @@ -18,22 +18,22 @@ use super::order::{ Inventory, Order, OrderFulfillmentSaga, OrderItem, OrderStatus, Payment, PaymentStatus, SagaStatus, }; -use sourced_rust::{AggregateBuilder, HashMapRepository}; +use sourced_rust::{AsyncAggregateBuilder, HashMapRepository}; -#[test] -fn saga_happy_path_completes_order() { +#[tokio::test] +async fn saga_happy_path_completes_order() { // Set up repositories for each aggregate type - let order_repo = HashMapRepository::new().aggregate::(); - let inventory_repo = HashMapRepository::new().aggregate::(); - let payment_repo = HashMapRepository::new().aggregate::(); - let saga_repo = HashMapRepository::new().aggregate::(); + let order_repo = HashMapRepository::new().async_aggregate::(); + let inventory_repo = HashMapRepository::new().async_aggregate::(); + let payment_repo = HashMapRepository::new().async_aggregate::(); + let saga_repo = HashMapRepository::new().async_aggregate::(); // === Setup: Initialize inventory === let mut widget_inventory = Inventory::new(); widget_inventory .initialize("WIDGET-001".to_string(), 100) .unwrap(); - inventory_repo.commit(&mut widget_inventory).unwrap(); + inventory_repo.commit(&mut widget_inventory).await.unwrap(); // === Step 1: Create Order === let order_id = "order-123".to_string(); @@ -47,7 +47,7 @@ fn saga_happy_path_completes_order() { order .create(order_id.clone(), "customer-456".to_string(), items.clone()) .unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Step 2: Start Saga === let mut order_fulfillment_saga = OrderFulfillmentSaga::new(); @@ -61,28 +61,28 @@ fn saga_happy_path_completes_order() { ) .unwrap(); assert_eq!(order_fulfillment_saga.status(), SagaStatus::Started); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Step 3: Reserve Inventory === - let mut inventory = inventory_repo.get("WIDGET-001").unwrap().unwrap(); + let mut inventory = inventory_repo.get("WIDGET-001").await.unwrap().unwrap(); assert!(inventory.can_reserve(5)); inventory.reserve(order_id.clone(), 5).unwrap(); - inventory_repo.commit(&mut inventory).unwrap(); + inventory_repo.commit(&mut inventory).await.unwrap(); // Update saga state - let mut order_fulfillment_saga = saga_repo.get("saga-123").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-123").await.unwrap().unwrap(); order_fulfillment_saga.inventory_reserved().unwrap(); assert_eq!( order_fulfillment_saga.status(), SagaStatus::InventoryReserved ); assert!(order_fulfillment_saga.compensation().inventory_reserved); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // Update order state - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order.mark_inventory_reserved().unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Step 4: Process Payment === let mut payment = Payment::new(); @@ -92,67 +92,67 @@ fn saga_happy_path_completes_order() { payment.authorize("txn-abc123".to_string()).unwrap(); payment.capture().unwrap(); assert!(payment.is_successful()); - payment_repo.commit(&mut payment).unwrap(); + payment_repo.commit(&mut payment).await.unwrap(); // Update saga state - let mut order_fulfillment_saga = saga_repo.get("saga-123").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-123").await.unwrap().unwrap(); order_fulfillment_saga.payment_succeeded().unwrap(); assert_eq!( order_fulfillment_saga.status(), SagaStatus::PaymentProcessed ); assert!(order_fulfillment_saga.compensation().payment_processed); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // Update order state - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order.mark_payment_processed().unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Step 5: Complete Saga === - let mut order_fulfillment_saga = saga_repo.get("saga-123").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-123").await.unwrap().unwrap(); order_fulfillment_saga.complete().unwrap(); assert_eq!(order_fulfillment_saga.status(), SagaStatus::Completed); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // Commit the inventory reservation (no longer reversible) - let mut inventory = inventory_repo.get("WIDGET-001").unwrap().unwrap(); + let mut inventory = inventory_repo.get("WIDGET-001").await.unwrap().unwrap(); inventory.commit_reservation(order_id.clone()).unwrap(); - inventory_repo.commit(&mut inventory).unwrap(); + inventory_repo.commit(&mut inventory).await.unwrap(); // Complete the order - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order.complete().unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Verify Final State === - let final_order_fulfillment_saga = saga_repo.get("saga-123").unwrap().unwrap(); + let final_order_fulfillment_saga = saga_repo.get("saga-123").await.unwrap().unwrap(); assert_eq!(final_order_fulfillment_saga.status(), SagaStatus::Completed); assert!(final_order_fulfillment_saga.is_complete()); - let final_order = order_repo.get(&order_id).unwrap().unwrap(); + let final_order = order_repo.get(&order_id).await.unwrap().unwrap(); assert_eq!(final_order.status(), OrderStatus::Completed); - let final_inventory = inventory_repo.get("WIDGET-001").unwrap().unwrap(); + let final_inventory = inventory_repo.get("WIDGET-001").await.unwrap().unwrap(); assert_eq!(final_inventory.available(), 95); // 100 - 5 assert_eq!(final_inventory.reserved(), 0); assert!(final_inventory.reservation_for_order(&order_id).is_none()); } -#[test] -fn saga_compensates_on_payment_failure() { +#[tokio::test] +async fn saga_compensates_on_payment_failure() { // Set up repositories - let order_repo = HashMapRepository::new().aggregate::(); - let inventory_repo = HashMapRepository::new().aggregate::(); - let payment_repo = HashMapRepository::new().aggregate::(); - let saga_repo = HashMapRepository::new().aggregate::(); + let order_repo = HashMapRepository::new().async_aggregate::(); + let inventory_repo = HashMapRepository::new().async_aggregate::(); + let payment_repo = HashMapRepository::new().async_aggregate::(); + let saga_repo = HashMapRepository::new().async_aggregate::(); // === Setup === let mut widget_inventory = Inventory::new(); widget_inventory .initialize("WIDGET-002".to_string(), 50) .unwrap(); - inventory_repo.commit(&mut widget_inventory).unwrap(); + inventory_repo.commit(&mut widget_inventory).await.unwrap(); let order_id = "order-fail-456".to_string(); let items = vec![OrderItem { @@ -165,7 +165,7 @@ fn saga_compensates_on_payment_failure() { order .create(order_id.clone(), "customer-789".to_string(), items.clone()) .unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Start Saga === let mut order_fulfillment_saga = OrderFulfillmentSaga::new(); @@ -178,23 +178,23 @@ fn saga_compensates_on_payment_failure() { 5000, ) .unwrap(); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Reserve Inventory (succeeds) === - let mut inventory = inventory_repo.get("WIDGET-002").unwrap().unwrap(); + let mut inventory = inventory_repo.get("WIDGET-002").await.unwrap().unwrap(); inventory.reserve(order_id.clone(), 10).unwrap(); - inventory_repo.commit(&mut inventory).unwrap(); + inventory_repo.commit(&mut inventory).await.unwrap(); - let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").await.unwrap().unwrap(); order_fulfillment_saga.inventory_reserved().unwrap(); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order.mark_inventory_reserved().unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // Verify inventory is reserved - let inventory = inventory_repo.get("WIDGET-002").unwrap().unwrap(); + let inventory = inventory_repo.get("WIDGET-002").await.unwrap().unwrap(); assert_eq!(inventory.available(), 40); // 50 - 10 assert_eq!(inventory.reserved(), 10); @@ -206,72 +206,72 @@ fn saga_compensates_on_payment_failure() { payment.fail("Insufficient funds".to_string()).unwrap(); assert!(!payment.is_successful()); assert_eq!(payment.status(), PaymentStatus::Failed); - payment_repo.commit(&mut payment).unwrap(); + payment_repo.commit(&mut payment).await.unwrap(); // === Saga enters compensation mode === - let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").await.unwrap().unwrap(); order_fulfillment_saga .step_failed("Payment".to_string(), "Insufficient funds".to_string()) .unwrap(); assert_eq!(order_fulfillment_saga.status(), SagaStatus::Compensating); assert!(order_fulfillment_saga.needs_inventory_compensation()); assert!(!order_fulfillment_saga.needs_payment_compensation()); // Payment wasn't successful - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Compensate: Release Inventory === - let mut inventory = inventory_repo.get("WIDGET-002").unwrap().unwrap(); + let mut inventory = inventory_repo.get("WIDGET-002").await.unwrap().unwrap(); inventory.release_reservation(order_id.clone()).unwrap(); - inventory_repo.commit(&mut inventory).unwrap(); + inventory_repo.commit(&mut inventory).await.unwrap(); - let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").await.unwrap().unwrap(); order_fulfillment_saga.inventory_compensated().unwrap(); assert!(!order_fulfillment_saga.needs_inventory_compensation()); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Cancel Order === - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order .cancel("Payment failed: Insufficient funds".to_string()) .unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Mark Saga as Failed === - let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-fail-456").await.unwrap().unwrap(); order_fulfillment_saga.mark_failed().unwrap(); assert_eq!(order_fulfillment_saga.status(), SagaStatus::Failed); assert!(order_fulfillment_saga.is_complete()); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Verify Final State === - let final_order_fulfillment_saga = saga_repo.get("saga-fail-456").unwrap().unwrap(); + let final_order_fulfillment_saga = saga_repo.get("saga-fail-456").await.unwrap().unwrap(); assert_eq!(final_order_fulfillment_saga.status(), SagaStatus::Failed); assert_eq!( final_order_fulfillment_saga.failure_reason(), Some("Payment: Insufficient funds") ); - let final_order = order_repo.get(&order_id).unwrap().unwrap(); + let final_order = order_repo.get(&order_id).await.unwrap().unwrap(); assert_eq!(final_order.status(), OrderStatus::Cancelled); // Inventory should be restored - let final_inventory = inventory_repo.get("WIDGET-002").unwrap().unwrap(); + let final_inventory = inventory_repo.get("WIDGET-002").await.unwrap().unwrap(); assert_eq!(final_inventory.available(), 50); // Back to original assert_eq!(final_inventory.reserved(), 0); } -#[test] -fn saga_compensates_on_inventory_failure() { +#[tokio::test] +async fn saga_compensates_on_inventory_failure() { // Set up repositories - let order_repo = HashMapRepository::new().aggregate::(); - let inventory_repo = HashMapRepository::new().aggregate::(); - let saga_repo = HashMapRepository::new().aggregate::(); + let order_repo = HashMapRepository::new().async_aggregate::(); + let inventory_repo = HashMapRepository::new().async_aggregate::(); + let saga_repo = HashMapRepository::new().async_aggregate::(); // === Setup: Low inventory === let mut widget_inventory = Inventory::new(); widget_inventory .initialize("WIDGET-003".to_string(), 5) .unwrap(); // Only 5 available - inventory_repo.commit(&mut widget_inventory).unwrap(); + inventory_repo.commit(&mut widget_inventory).await.unwrap(); let order_id = "order-inv-fail-789".to_string(); let items = vec![OrderItem { @@ -284,7 +284,7 @@ fn saga_compensates_on_inventory_failure() { order .create(order_id.clone(), "customer-xyz".to_string(), items.clone()) .unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // === Start Saga === let mut order_fulfillment_saga = OrderFulfillmentSaga::new(); @@ -297,14 +297,14 @@ fn saga_compensates_on_inventory_failure() { 5000, ) .unwrap(); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Try to Reserve Inventory (fails - not enough stock) === - let inventory = inventory_repo.get("WIDGET-003").unwrap().unwrap(); + let inventory = inventory_repo.get("WIDGET-003").await.unwrap().unwrap(); assert!(!inventory.can_reserve(10)); // Can't reserve 10 when only 5 available // Saga fails at first step - let mut order_fulfillment_saga = saga_repo.get("saga-inv-fail").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-inv-fail").await.unwrap().unwrap(); order_fulfillment_saga .step_failed( "Inventory".to_string(), @@ -315,35 +315,35 @@ fn saga_compensates_on_inventory_failure() { // No compensation needed - nothing was reserved yet assert!(!order_fulfillment_saga.needs_inventory_compensation()); assert!(!order_fulfillment_saga.needs_payment_compensation()); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // Cancel order - let mut order = order_repo.get(&order_id).unwrap().unwrap(); + let mut order = order_repo.get(&order_id).await.unwrap().unwrap(); order.cancel("Insufficient stock".to_string()).unwrap(); - order_repo.commit(&mut order).unwrap(); + order_repo.commit(&mut order).await.unwrap(); // Mark saga as failed (no compensation needed) - let mut order_fulfillment_saga = saga_repo.get("saga-inv-fail").unwrap().unwrap(); + let mut order_fulfillment_saga = saga_repo.get("saga-inv-fail").await.unwrap().unwrap(); order_fulfillment_saga.mark_failed().unwrap(); - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // === Verify Final State === - let final_order_fulfillment_saga = saga_repo.get("saga-inv-fail").unwrap().unwrap(); + let final_order_fulfillment_saga = saga_repo.get("saga-inv-fail").await.unwrap().unwrap(); assert_eq!(final_order_fulfillment_saga.status(), SagaStatus::Failed); assert!(final_order_fulfillment_saga.is_complete()); - let final_order = order_repo.get(&order_id).unwrap().unwrap(); + let final_order = order_repo.get(&order_id).await.unwrap().unwrap(); assert_eq!(final_order.status(), OrderStatus::Cancelled); // Inventory unchanged - let final_inventory = inventory_repo.get("WIDGET-003").unwrap().unwrap(); + let final_inventory = inventory_repo.get("WIDGET-003").await.unwrap().unwrap(); assert_eq!(final_inventory.available(), 5); assert_eq!(final_inventory.reserved(), 0); } -#[test] -fn saga_is_replayable_from_events() { - let saga_repo = HashMapRepository::new().aggregate::(); +#[tokio::test] +async fn saga_is_replayable_from_events() { + let saga_repo = HashMapRepository::new().async_aggregate::(); let items = vec![OrderItem { sku: "WIDGET-REPLAY".to_string(), @@ -366,10 +366,10 @@ fn saga_is_replayable_from_events() { order_fulfillment_saga.payment_succeeded().unwrap(); // Commit to repository - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); // Retrieve and verify state is reconstructed from events - let restored = saga_repo.get("saga-replay").unwrap().unwrap(); + let restored = saga_repo.get("saga-replay").await.unwrap().unwrap(); assert_eq!(restored.order_id(), "order-replay"); assert_eq!(restored.customer_id(), "customer-replay"); @@ -381,15 +381,15 @@ fn saga_is_replayable_from_events() { // Can continue from restored state let mut restored = restored; restored.complete().unwrap(); - saga_repo.commit(&mut restored).unwrap(); + saga_repo.commit(&mut restored).await.unwrap(); - let final_order_fulfillment_saga = saga_repo.get("saga-replay").unwrap().unwrap(); + let final_order_fulfillment_saga = saga_repo.get("saga-replay").await.unwrap().unwrap(); assert_eq!(final_order_fulfillment_saga.status(), SagaStatus::Completed); } -#[test] -fn saga_tracks_compensation_state_correctly() { - let saga_repo = HashMapRepository::new().aggregate::(); +#[tokio::test] +async fn saga_tracks_compensation_state_correctly() { + let saga_repo = HashMapRepository::new().async_aggregate::(); let items = vec![OrderItem { sku: "WIDGET-COMP".to_string(), @@ -447,8 +447,8 @@ fn saga_tracks_compensation_state_correctly() { assert!(order_fulfillment_saga.is_complete()); // Verify it persists correctly - saga_repo.commit(&mut order_fulfillment_saga).unwrap(); - let restored = saga_repo.get("saga-comp").unwrap().unwrap(); + saga_repo.commit(&mut order_fulfillment_saga).await.unwrap(); + let restored = saga_repo.get("saga-comp").await.unwrap().unwrap(); assert_eq!(restored.status(), SagaStatus::Failed); assert!(!restored.compensation().inventory_reserved); assert!(!restored.compensation().payment_processed); diff --git a/tests/snapshots/main.rs b/tests/snapshots/main.rs index 057e9f49d..0fa1511a1 100644 --- a/tests/snapshots/main.rs +++ b/tests/snapshots/main.rs @@ -3,8 +3,8 @@ mod aggregate; use aggregate::Todo; use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Aggregate, AggregateBuilder, Entity, HashMapRepository, Queueable, SnapshotRecord, - SnapshotStore, Snapshottable, + sourced, Aggregate, AsyncAggregateBuilder, AsyncSnapshotStore, Entity, HashMapRepository, + Queueable, SnapshotRecord, Snapshottable, StreamIdentity, }; #[derive(Default)] @@ -46,27 +46,40 @@ impl Snapshottable for ReplayCounter { } } -#[test] -fn snapshot_created_at_frequency_threshold() { +#[tokio::test] +async fn snapshot_created_at_frequency_threshold() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Version 1 — below threshold of 2, no snapshot yet - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_none()); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .is_none()); // Load, add another event to reach version 2 - let mut todo = repo.get("t1").unwrap().unwrap(); + let mut todo = repo.get("t1").await.unwrap().unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); // Version 2 >= 0 + 2 — snapshot should now exist - let snap = repo.repo().repo().get_snapshot("t1").unwrap(); + let snap = repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap(); assert!(snap.is_some()); let snap = snap.unwrap(); assert_eq!(snap.version, 2); @@ -75,7 +88,7 @@ fn snapshot_created_at_frequency_threshold() { assert_eq!(snap.payload_codec, sourced_rust::BITCODE_PAYLOAD_CODEC); // Reload and verify state - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); let s = loaded.snapshot(); assert_eq!(s.id, "t1"); assert_eq!(s.user_id, "alice"); @@ -85,38 +98,54 @@ fn snapshot_created_at_frequency_threshold() { assert_eq!(loaded.entity.snapshot_version(), 2); } -#[test] -fn no_snapshot_before_threshold() { +#[tokio::test] +async fn no_snapshot_before_threshold() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(5); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Only 1 event, threshold is 5 - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_none()); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .is_none()); } -#[test] -fn load_from_snapshot_produces_correct_state() { +#[tokio::test] +async fn load_from_snapshot_produces_correct_state() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot at version 2 - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_some()); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .is_some()); // Reload — should use snapshot - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); let snap = loaded.snapshot(); assert_eq!(snap.id, "t1"); assert_eq!(snap.user_id, "alice"); @@ -124,74 +153,88 @@ fn load_from_snapshot_produces_correct_state() { assert!(snap.completed); } -#[test] -fn snapshot_plus_newer_events() { +#[tokio::test] +async fn snapshot_plus_newer_events() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); // Create and commit 2 events (triggers snapshot at version 2) let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let mut todo = repo.get("t1").unwrap().unwrap(); + let mut todo = repo.get("t1").await.unwrap().unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot exists at version 2, completed = true - let snap = repo.repo().repo().get_snapshot("t1").unwrap().unwrap(); + let snap = repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .unwrap(); assert_eq!(snap.version, 2); // Now create a second todo to verify snapshot + partial replay works. // We'll use a different approach: create a fresh repo pointing to the same storage // and verify loading still works. - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert!(loaded.snapshot().completed); assert_eq!(loaded.entity.version(), 2); assert_eq!(loaded.entity.snapshot_version(), 2); } -#[test] -fn snapshot_hydration_replays_every_event_after_snapshot_version() { +#[tokio::test] +async fn snapshot_hydration_replays_every_event_after_snapshot_version() { let base_repo = HashMapRepository::new(); - let full_replay_repo = base_repo.clone().aggregate::(); + let full_replay_repo = base_repo.clone().async_aggregate::(); let snapshot_repo = base_repo .clone() - .aggregate::() + .async_aggregate::() .with_snapshots(100); let mut counter = ReplayCounter::default(); counter.add("counter-1".into(), 10).unwrap(); - full_replay_repo.commit(&mut counter).unwrap(); + full_replay_repo.commit(&mut counter).await.unwrap(); let payload = bitcode::serialize(&ReplayCounterSnapshot { id: "counter-1".into(), total: 10, }) .unwrap(); + let counter_identity = + StreamIdentity::new(ReplayCounter::aggregate_type(), "counter-1").unwrap(); base_repo - .save_snapshot(SnapshotRecord::new( - ReplayCounter::aggregate_type(), - "counter-1", - 1, - std::any::type_name::(), - 1, - payload, - )) + .save_snapshot_async( + &counter_identity, + SnapshotRecord::new( + ReplayCounter::aggregate_type(), + "counter-1", + 1, + std::any::type_name::(), + 1, + payload, + ), + ) + .await .unwrap(); - let mut counter = snapshot_repo.get("counter-1").unwrap().unwrap(); + let mut counter = snapshot_repo.get("counter-1").await.unwrap().unwrap(); counter.add("counter-1".into(), 5).unwrap(); - snapshot_repo.commit(&mut counter).unwrap(); + snapshot_repo.commit(&mut counter).await.unwrap(); - let mut counter = snapshot_repo.get("counter-1").unwrap().unwrap(); + let mut counter = snapshot_repo.get("counter-1").await.unwrap().unwrap(); counter.add("counter-1".into(), 7).unwrap(); - snapshot_repo.commit(&mut counter).unwrap(); + snapshot_repo.commit(&mut counter).await.unwrap(); - let loaded = snapshot_repo.get("counter-1").unwrap().unwrap(); - let replayed = full_replay_repo.get("counter-1").unwrap().unwrap(); + let loaded = snapshot_repo.get("counter-1").await.unwrap().unwrap(); + let replayed = full_replay_repo.get("counter-1").await.unwrap().unwrap(); assert_eq!(loaded.total, 22); assert_eq!(loaded.total, replayed.total); @@ -211,94 +254,134 @@ fn snapshot_hydration_replays_every_event_after_snapshot_version() { ); } -#[test] -fn no_snapshot_falls_back_to_full_replay() { +#[tokio::test] +async fn no_snapshot_falls_back_to_full_replay() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot exists - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_some()); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .is_some()); // Delete the snapshot - repo.repo().repo().delete_snapshot("t1").unwrap(); - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_none()); + repo.repo() + .repo() + .delete_snapshot_async(&identity) + .await + .unwrap(); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .is_none()); // Loading should still work via full replay - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); let snap = loaded.snapshot(); assert_eq!(snap.id, "t1"); assert_eq!(snap.user_id, "alice"); assert!(snap.completed); } -#[test] -fn snapshot_version_advances_on_second_snapshot() { +#[tokio::test] +async fn snapshot_version_advances_on_second_snapshot() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(1); // snapshot every event let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // First snapshot at version 1 - let snap = repo.repo().repo().get_snapshot("t1").unwrap().unwrap(); + let snap = repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .unwrap(); assert_eq!(snap.version, 1); // Add another event - let mut todo = repo.get("t1").unwrap().unwrap(); + let mut todo = repo.get("t1").await.unwrap().unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); // Second snapshot at version 2 - let snap = repo.repo().repo().get_snapshot("t1").unwrap().unwrap(); + let snap = repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap() + .unwrap(); assert_eq!(snap.version, 2); // Verify the loaded aggregate has correct snapshot_version - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.entity.snapshot_version(), 2); assert!(loaded.snapshot().completed); } -#[test] -fn with_queued_repo() { +#[tokio::test] +async fn with_queued_repo() { let repo = HashMapRepository::new() - .queued() - .aggregate::() + .queued_async() + .async_aggregate::() .with_snapshots(2); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let mut todo = repo.get("t1").unwrap().unwrap(); + let mut todo = repo.get("t1").await.unwrap().unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot should exist through the queued + snapshot chain - let snap = repo.repo().repo().inner().get_snapshot("t1").unwrap(); + let snap = repo + .repo() + .repo() + .inner() + .get_snapshot_async(&identity) + .await + .unwrap(); assert!(snap.is_some()); assert_eq!(snap.unwrap().version, 2); // Reload and verify - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert!(loaded.snapshot().completed); assert_eq!(loaded.entity.snapshot_version(), 2); } -#[test] -fn get_all_with_snapshots() { +#[tokio::test] +async fn get_all_with_snapshots() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); // Create two todos, both past snapshot threshold @@ -307,28 +390,28 @@ fn get_all_with_snapshots() { .initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); todo1.complete().unwrap(); - repo.commit(&mut todo1).unwrap(); + repo.commit(&mut todo1).await.unwrap(); let mut todo2 = Todo::new(); todo2 .initialize("t2".into(), "bob".into(), "Walk dog".into()) .unwrap(); todo2.complete().unwrap(); - repo.commit(&mut todo2).unwrap(); + repo.commit(&mut todo2).await.unwrap(); - let todos = repo.get_all(&["t1", "t2"]).unwrap(); + let todos = repo.get_all(&["t1", "t2"]).await.unwrap(); assert_eq!(todos.len(), 2); assert!(todos.iter().all(|todo| todo.snapshot().completed)); - let alice = repo.get("t1").unwrap().unwrap(); + let alice = repo.get("t1").await.unwrap().unwrap(); assert_eq!(alice.snapshot().user_id, "alice"); assert_eq!(alice.snapshot().task, "Buy milk"); } -#[test] -fn commit_all_with_snapshots() { +#[tokio::test] +async fn commit_all_with_snapshots() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); let mut todo1 = Todo::new(); @@ -343,11 +426,28 @@ fn commit_all_with_snapshots() { .unwrap(); todo2.complete().unwrap(); // version 2 - repo.commit_all(&mut [&mut todo1, &mut todo2]).unwrap(); + repo.commit_all(&mut [&mut todo1, &mut todo2]) + .await + .unwrap(); + + let identity1 = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); + let identity2 = StreamIdentity::new(Todo::aggregate_type(), "t2").unwrap(); // Both should have snapshots at version 2 - let snap1 = repo.repo().repo().get_snapshot("t1").unwrap().unwrap(); + let snap1 = repo + .repo() + .repo() + .get_snapshot_async(&identity1) + .await + .unwrap() + .unwrap(); assert_eq!(snap1.version, 2); - let snap2 = repo.repo().repo().get_snapshot("t2").unwrap().unwrap(); + let snap2 = repo + .repo() + .repo() + .get_snapshot_async(&identity2) + .await + .unwrap() + .unwrap(); assert_eq!(snap2.version, 2); } diff --git a/tests/sourced/main.rs b/tests/sourced/main.rs index baf609326..9924baadf 100644 --- a/tests/sourced/main.rs +++ b/tests/sourced/main.rs @@ -4,7 +4,7 @@ use aggregate::{Todo, TodoEvent}; use serde::ser::Error as _; use serde::Serialize; use sourced_rust::{ - Aggregate, AggregateBuilder, Entity, EventRecord, EventRecordError, HashMapRepository, + Aggregate, AsyncAggregateBuilder, Entity, EventRecord, EventRecordError, HashMapRepository, Queueable, }; @@ -196,18 +196,20 @@ fn try_from_unknown_event_returns_error() { assert!(result.unwrap_err().contains("Unknown event")); } -#[test] -fn aggregate_hydration_roundtrip() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn aggregate_hydration_roundtrip() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut todo = Todo::default(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().id, "t1"); assert_eq!(loaded.snapshot().user_id, "alice"); assert_eq!(loaded.snapshot().task, "Buy milk"); diff --git a/tests/sourced_enqueue/main.rs b/tests/sourced_enqueue/main.rs index d0c03691f..b35a250c6 100644 --- a/tests/sourced_enqueue/main.rs +++ b/tests/sourced_enqueue/main.rs @@ -1,7 +1,7 @@ mod aggregate; use aggregate::{Notifier, NotifierEvent, Order, OrderEvent}; -use sourced_rust::{Aggregate, AggregateBuilder, HashMapRepository, Queueable}; +use sourced_rust::{Aggregate, AsyncAggregateBuilder, HashMapRepository, Queueable}; use std::sync::mpsc; use std::time::Duration; @@ -34,18 +34,20 @@ fn full_lifecycle_digest_and_enqueue() { // Replay does not re-enqueue // ============================================================================= -#[test] -fn replay_does_not_re_enqueue() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn replay_does_not_re_enqueue() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut order = Order::default(); order.create("order-1".into(), "alice".into()).unwrap(); order.confirm().unwrap(); order.emitter.emit_queued(); - repo.commit(&mut order).unwrap(); + repo.commit(&mut order).await.unwrap(); - let loaded = repo.get("order-1").unwrap().unwrap(); + let loaded = repo.get("order-1").await.unwrap().unwrap(); assert_eq!(loaded.emitter.queued_len(), 0); assert_eq!(loaded.status, "confirmed"); assert_eq!(loaded.entity.version(), 2); @@ -132,16 +134,18 @@ fn custom_emitter_field_emits() { .expect("NotificationSent callback never fired"); } -#[test] -fn custom_emitter_replay_does_not_enqueue() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn custom_emitter_replay_does_not_enqueue() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut notifier = Notifier::default(); notifier.send("n-1".into(), "Hello".into()).unwrap(); notifier.my_emitter.emit_queued(); - repo.commit(&mut notifier).unwrap(); + repo.commit(&mut notifier).await.unwrap(); - let loaded = repo.get("n-1").unwrap().unwrap(); + let loaded = repo.get("n-1").await.unwrap().unwrap(); assert_eq!(loaded.my_emitter.queued_len(), 0); assert_eq!(loaded.message, "Hello"); } diff --git a/tests/sourced_snapshot/main.rs b/tests/sourced_snapshot/main.rs index 5880c46d6..7bf7339bf 100644 --- a/tests/sourced_snapshot/main.rs +++ b/tests/sourced_snapshot/main.rs @@ -2,8 +2,8 @@ mod aggregates; use aggregates::*; use sourced_rust::{ - AggregateBuilder, HashMapRepository, OutboxMessage, OutboxStore, SnapshotStore, Snapshottable, - SyncOutboxCommitExt, + Aggregate, AsyncAggregateBuilder, AsyncSnapshotStore, HashMapRepository, OutboxMessage, + OutboxStore, Snapshottable, StreamIdentity, }; // ============================================================================ @@ -24,18 +24,18 @@ fn default_snapshot_has_id_and_all_fields() { assert!(snap.completed); } -#[test] -fn default_snapshot_roundtrip_via_snapshottable() { +#[tokio::test] +async fn default_snapshot_roundtrip_via_snapshottable() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(1); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Buy milk".into()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); let snap = loaded.snapshot(); assert_eq!(snap.id, "t1"); assert_eq!(snap.user_id, "alice"); @@ -87,17 +87,17 @@ fn custom_id_restore_sets_entity_id_from_field() { assert_eq!(inv.available, 50); } -#[test] -fn custom_id_roundtrip_via_repo() { +#[tokio::test] +async fn custom_id_roundtrip_via_repo() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(1); let mut inv = Inventory::new(); inv.create("inv-1".into(), "SKU-A".into(), 10).unwrap(); - repo.commit(&mut inv).unwrap(); + repo.commit(&mut inv).await.unwrap(); - let loaded = repo.get("inv-1").unwrap().unwrap(); + let loaded = repo.get("inv-1").await.unwrap().unwrap(); let snap = loaded.snapshot(); assert_eq!(snap.sku, "SKU-A"); assert_eq!(snap.available, 10); @@ -130,17 +130,17 @@ fn serde_skip_default_excluded_from_snapshot() { // emitter is not in NotifierSnapshot - verified by compilation } -#[test] -fn serde_skip_restore_roundtrip() { +#[tokio::test] +async fn serde_skip_restore_roundtrip() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(1); let mut order = Order::new(); order.place("o1".into(), "alice".into(), 500).unwrap(); - repo.commit(&mut order).unwrap(); + repo.commit(&mut order).await.unwrap(); - let loaded = repo.get("o1").unwrap().unwrap(); + let loaded = repo.get("o1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().customer, "alice"); assert_eq!(loaded.snapshot().total, 500); // cached_label will be default (empty) after restore, which is correct @@ -163,22 +163,28 @@ fn sourced_attr_with_snapshot_derive() { assert_eq!(snap.count, 8); } -#[test] -fn sourced_attr_snapshot_roundtrip_via_repo() { +#[tokio::test] +async fn sourced_attr_snapshot_roundtrip_via_repo() { let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(2); let mut counter = Counter::new(); counter.initialize("c1".into()).unwrap(); counter.increment(10).unwrap(); - repo.commit(&mut counter).unwrap(); + repo.commit(&mut counter).await.unwrap(); // At version 2, should have a snapshot - let snap_record = repo.repo().repo().get_snapshot("c1").unwrap(); + let identity = StreamIdentity::new(Counter::aggregate_type(), "c1").unwrap(); + let snap_record = repo + .repo() + .repo() + .get_snapshot_async(&identity) + .await + .unwrap(); assert!(snap_record.is_some()); - let loaded = repo.get("c1").unwrap().unwrap(); + let loaded = repo.get("c1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().count, 10); } @@ -246,18 +252,18 @@ fn domain_event_propagates_metadata() { assert_eq!(outbox.meta("user_id"), Some("u-42")); } -#[test] -fn domain_event_commits_with_outbox() { - let repo = HashMapRepository::new().aggregate::(); +#[tokio::test] +async fn domain_event_commits_with_outbox() { + let repo = HashMapRepository::new().async_aggregate::(); let mut todo = Todo::new(); todo.initialize("t1".into(), "alice".into(), "Ship it".into()) .unwrap(); let outbox = OutboxMessage::domain_event("TodoInitialized", &todo).unwrap(); - repo.outbox_sync(outbox).commit_sync(&mut todo).unwrap(); + repo.outbox(outbox).commit(&mut todo).await.unwrap(); - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().task, "Ship it"); let pending = repo.repo().outbox_store().pending().unwrap(); assert_eq!(pending.len(), 1); diff --git a/tests/todos/main.rs b/tests/todos/main.rs index f3072bd60..618e6a096 100644 --- a/tests/todos/main.rs +++ b/tests/todos/main.rs @@ -2,10 +2,9 @@ mod aggregate; use aggregate::{Todo, TodoSnapshot}; use sourced_rust::{ - AggregateBuilder, ClaimOutboxMessages, Commit, EventEmitter, GetAggregate, HashMapRepository, - LocalEmitterPublisher, LockError, LogPublisher, OutboxClaimRef, OutboxMessage, - OutboxMessageStatus, OutboxStore, OutboxWorker, Queueable, RepositoryError, - SyncCommitBuilderExt, SyncOutboxCommitExt, + AsyncAggregateBuilder, AsyncCommitBuilderExt, AsyncLock, AsyncLockManager, ClaimOutboxMessages, + EventEmitter, HashMapRepository, LocalEmitterPublisher, LogPublisher, OutboxClaimRef, + OutboxMessage, OutboxMessageStatus, OutboxStore, OutboxWorker, Queueable, RepositoryError, }; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Arc, Mutex}; @@ -52,9 +51,11 @@ fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { panic!("outbox message `{id}` should exist") } -#[test] -fn todos() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn todos() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); // Create a new Todo + Outbox messages let mut todo = Todo::new(); @@ -72,8 +73,9 @@ fn todos() { .unwrap(); // Commit the Todo + Outbox message to the repository - repo.outbox_sync(init_message) - .commit_sync(&mut todo) + repo.outbox(init_message) + .commit(&mut todo) + .await .expect("initial todo outbox commit should succeed"); // Verify the outbox event was captured @@ -84,7 +86,7 @@ fn todos() { } // Retrieve the Todo from the repository and complete it, then commit again - if let Some(mut retrieved_todo) = repo.get(&id1).unwrap() { + if let Some(mut retrieved_todo) = repo.get(&id1).await.unwrap() { retrieved_todo.complete().unwrap(); // Add an outbox event for the completion @@ -95,8 +97,9 @@ fn todos() { ) .unwrap(); - repo.outbox_sync(complete_message) - .commit_sync(&mut retrieved_todo) + repo.outbox(complete_message) + .commit(&mut retrieved_todo) + .await .expect("completed todo outbox commit should succeed"); // Verify we now have 2 outbox events @@ -109,7 +112,7 @@ fn todos() { assert!(pending.iter().any(|msg| msg.event_type == "TodoCompleted")); } - if let Some(completed_todo) = repo.get(&id1).unwrap() { + if let Some(completed_todo) = repo.get(&id1).await.unwrap() { assert!(completed_todo.snapshot().id == id1); assert!(completed_todo.snapshot().user_id == "user1"); assert!(completed_todo.snapshot().task == "Buy groceries"); @@ -140,10 +143,10 @@ fn todos() { .unwrap(); // Commit multiple Todos to the repository - let _ = repo.commit_all(&mut [&mut todo2, &mut todo3]); + let _ = repo.commit_all(&mut [&mut todo2, &mut todo3]).await; // get all the todos from the repository - let all_todos = repo.peek_all(&[&id1, &id2, &id3]).unwrap(); + let all_todos = repo.peek_all(&[&id1, &id2, &id3]).await.unwrap(); if !all_todos.is_empty() { assert!(all_todos.len() == 3); } else { @@ -151,26 +154,30 @@ fn todos() { } } -#[test] -fn get_commit_roundtrip() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn get_commit_roundtrip() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut todo = Todo::new(); let id = next_id(); todo.initialize(id.clone(), "user1".to_string(), "Roundtrip".to_string()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let retrieved = repo.peek(&id).unwrap().expect("Todo not found"); + let retrieved = repo.peek(&id).await.unwrap().expect("Todo not found"); assert_eq!(retrieved.snapshot().id, id); assert_eq!(retrieved.snapshot().user_id, "user1"); assert_eq!(retrieved.snapshot().task, "Roundtrip"); assert!(!retrieved.snapshot().completed); } -#[test] -fn get_all_commit_all_roundtrip() { - let repo = HashMapRepository::new().queued().aggregate::(); +#[tokio::test] +async fn get_all_commit_all_roundtrip() { + let repo = HashMapRepository::new() + .queued_async() + .async_aggregate::(); let mut todo1 = Todo::new(); let id1 = next_id(); @@ -184,9 +191,11 @@ fn get_all_commit_all_roundtrip() { .initialize(id2.clone(), "user2".to_string(), "Second".to_string()) .unwrap(); - repo.commit_all(&mut [&mut todo1, &mut todo2]).unwrap(); + repo.commit_all(&mut [&mut todo1, &mut todo2]) + .await + .unwrap(); - let todos = repo.get_all(&[&id1, &id2]).unwrap(); + let todos = repo.get_all(&[&id1, &id2]).await.unwrap(); assert_eq!(todos.len(), 2); assert_eq!(todos[0].snapshot().id, id1); assert!(!todos[0].snapshot().completed); @@ -200,9 +209,11 @@ fn get_all_commit_all_roundtrip() { todo1v2.complete().unwrap(); todo2v2.complete().unwrap(); - repo.commit_all(&mut [&mut todo1v2, &mut todo2v2]).unwrap(); + repo.commit_all(&mut [&mut todo1v2, &mut todo2v2]) + .await + .unwrap(); - let v2_todos = repo.peek_all(&[&id1, &id2]).unwrap(); + let v2_todos = repo.peek_all(&[&id1, &id2]).await.unwrap(); assert_eq!(v2_todos.len(), 2); assert_eq!(v2_todos[0].snapshot().id, id1); @@ -211,8 +222,8 @@ fn get_all_commit_all_roundtrip() { assert!(v2_todos[1].snapshot().completed); } -#[test] -fn outbox_records_persisted() { +#[tokio::test] +async fn outbox_records_persisted() { let repo = HashMapRepository::new(); let mut todo = Todo::new(); let id = next_id(); @@ -222,7 +233,7 @@ fn outbox_records_persisted() { let message = OutboxMessage::encode(format!("{}:init", id), "TodoInitialized", &snapshot).unwrap(); - repo.outbox_sync(message).commit_sync(&mut todo).unwrap(); + repo.outbox(message).commit(&mut todo).await.unwrap(); // Check pending outbox messages let pending = repo.outbox_store().pending().unwrap(); @@ -236,8 +247,8 @@ fn outbox_records_persisted() { assert_eq!(published.completed, snapshot.completed); } -#[test] -fn outbox_worker_log_publisher() { +#[tokio::test] +async fn outbox_worker_log_publisher() { let repo = HashMapRepository::new(); let mut todo = Todo::new(); let id = next_id(); @@ -251,7 +262,7 @@ fn outbox_worker_log_publisher() { let message = OutboxMessage::encode(format!("{}:init", id), "TodoInitialized", &snapshot).unwrap(); let message_id = message.id().to_string(); - repo.outbox_sync(message).commit_sync(&mut todo).unwrap(); + repo.outbox(message).commit(&mut todo).await.unwrap(); // Create worker with new API let buffer = Arc::new(Mutex::new(Vec::new())); @@ -288,8 +299,8 @@ fn outbox_worker_log_publisher() { assert!(published.is_published()); } -#[test] -fn outbox_worker_local_emitter_publisher() { +#[tokio::test] +async fn outbox_worker_local_emitter_publisher() { let repo = HashMapRepository::new(); let mut todo = Todo::new(); let id = next_id(); @@ -302,7 +313,7 @@ fn outbox_worker_local_emitter_publisher() { let snapshot = todo.snapshot(); let message = OutboxMessage::encode(format!("{}:init", id), "TodoInitialized", &snapshot).unwrap(); - repo.outbox_sync(message).commit_sync(&mut todo).unwrap(); + repo.outbox(message).commit(&mut todo).await.unwrap(); let mut emitter = EventEmitter::new(); let (tx, rx) = mpsc::channel::(); @@ -339,16 +350,20 @@ fn outbox_worker_local_emitter_publisher() { assert!(!payload.is_empty()); } -#[test] -fn abort_releases_lock_after_get() { - let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); +#[tokio::test] +async fn abort_releases_lock_after_get() { + let repo = Arc::new( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ); let mut todo = Todo::new(); let id = next_id(); todo.initialize(id.clone(), "user1".to_string(), "Abort get".to_string()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let locked = repo.get(&id).unwrap().unwrap(); + let locked = repo.get(&id).await.unwrap().unwrap(); let (tx_started, rx_started) = mpsc::channel(); let (tx_got, rx_got) = mpsc::channel(); @@ -356,7 +371,8 @@ fn abort_releases_lock_after_get() { let id_other = id.clone(); thread::spawn(move || { tx_started.send(()).unwrap(); - let _ = repo_other.get(&id_other).unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let _ = rt.block_on(repo_other.get(&id_other)).unwrap(); tx_got.send(()).unwrap(); }); @@ -367,9 +383,13 @@ fn abort_releases_lock_after_get() { assert!(rx_got.recv_timeout(Duration::from_millis(500)).is_ok()); } -#[test] -fn abort_releases_lock_after_get_all() { - let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); +#[tokio::test] +async fn abort_releases_lock_after_get_all() { + let repo = Arc::new( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ); let mut todo1 = Todo::new(); let id1 = next_id(); todo1 @@ -379,7 +399,7 @@ fn abort_releases_lock_after_get_all() { "Abort get_all 1".to_string(), ) .unwrap(); - repo.commit(&mut todo1).unwrap(); + repo.commit(&mut todo1).await.unwrap(); let mut todo2 = Todo::new(); let id2 = next_id(); @@ -390,9 +410,9 @@ fn abort_releases_lock_after_get_all() { "Abort get_all 2".to_string(), ) .unwrap(); - repo.commit(&mut todo2).unwrap(); + repo.commit(&mut todo2).await.unwrap(); - let locked = repo.get_all(&[&id1, &id2]).unwrap(); + let locked = repo.get_all(&[&id1, &id2]).await.unwrap(); let (tx_started, rx_started) = mpsc::channel(); let (tx_got, rx_got) = mpsc::channel(); @@ -400,7 +420,8 @@ fn abort_releases_lock_after_get_all() { let id_other = id1.clone(); thread::spawn(move || { tx_started.send(()).unwrap(); - let _ = repo_other.get(&id_other).unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let _ = rt.block_on(repo_other.get(&id_other)).unwrap(); tx_got.send(()).unwrap(); }); @@ -414,14 +435,18 @@ fn abort_releases_lock_after_get_all() { assert!(rx_got.recv_timeout(Duration::from_millis(500)).is_ok()); } -#[test] -fn queued_repo_blocks_get_until_commit() { - let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); +#[tokio::test] +async fn queued_repo_blocks_get_until_commit() { + let repo = Arc::new( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ); let mut todo = Todo::new(); let id = next_id(); todo.initialize(id.clone(), "user1".to_string(), "Queue test".to_string()) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); let mut other_todo = Todo::new(); let other_id = next_id(); @@ -432,7 +457,7 @@ fn queued_repo_blocks_get_until_commit() { "Independent queue".to_string(), ) .unwrap(); - repo.commit(&mut other_todo).unwrap(); + repo.commit(&mut other_todo).await.unwrap(); let (tx_started, rx_started) = mpsc::channel(); let (tx_release, rx_release) = mpsc::channel(); @@ -441,10 +466,11 @@ fn queued_repo_blocks_get_until_commit() { let repo_a = Arc::clone(&repo); let id_a = id.clone(); thread::spawn(move || { - let mut todo = repo_a.get(&id_a).unwrap().unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut todo = rt.block_on(repo_a.get(&id_a)).unwrap().unwrap(); tx_started.send(()).unwrap(); rx_release.recv().unwrap(); - let _ = repo_a.commit(&mut todo); + let _ = rt.block_on(repo_a.commit(&mut todo)); tx_committed.send(()).unwrap(); }); @@ -454,7 +480,11 @@ fn queued_repo_blocks_get_until_commit() { let repo_other = Arc::clone(&repo); let other_id_clone = other_id.clone(); thread::spawn(move || { - let todo = repo_other.get(&other_id_clone).unwrap().unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let todo = rt + .block_on(repo_other.get(&other_id_clone)) + .unwrap() + .unwrap(); repo_other.abort(&todo).unwrap(); tx_other_done.send(()).unwrap(); }); @@ -463,7 +493,8 @@ fn queued_repo_blocks_get_until_commit() { let repo_peek = Arc::clone(&repo); let id_peek = id.clone(); thread::spawn(move || { - let _ = repo_peek.peek(&id_peek).unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let _ = rt.block_on(repo_peek.peek(&id_peek)).unwrap(); tx_peek_done.send(()).unwrap(); }); @@ -471,8 +502,9 @@ fn queued_repo_blocks_get_until_commit() { let repo_peek_all = Arc::clone(&repo); let id_peek_all = id.clone(); thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); let ids = [id_peek_all.as_str()]; - let _ = repo_peek_all.peek_all(&ids).unwrap(); + let _ = rt.block_on(repo_peek_all.peek_all(&ids)).unwrap(); tx_peek_all_done.send(()).unwrap(); }); @@ -490,8 +522,9 @@ fn queued_repo_blocks_get_until_commit() { let repo_b = Arc::clone(&repo); let id_b = id.clone(); thread::spawn(move || { - let mut todo = repo_b.get(&id_b).unwrap().unwrap(); - let _ = repo_b.commit(&mut todo); + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut todo = rt.block_on(repo_b.get(&id_b)).unwrap().unwrap(); + let _ = rt.block_on(repo_b.commit(&mut todo)); tx_done.send(()).unwrap(); }); @@ -503,23 +536,34 @@ fn queued_repo_blocks_get_until_commit() { #[test] fn manual_lock_reports_failure_when_already_held() { - let repo = HashMapRepository::new().queued(); + let repo = HashMapRepository::new().queued_async(); let id = next_id(); - repo.lock(&id).unwrap(); - let err = repo.lock(&id).expect_err("second manual lock should fail"); - let is_lock_failure = matches!( - &err, - RepositoryError::Lock(LockError::AcquireFailed(message)) if message.contains(&id) + let lock = repo.lock_manager().get_lock(&id).unwrap(); + + // First acquisition succeeds. + assert!( + lock.try_lock().unwrap(), + "first manual lock should be acquired" + ); + + // Second acquisition reports the lock is already held. + let second = repo.lock_manager().get_lock(&id).unwrap(); + assert!( + !second.try_lock().unwrap(), + "second manual lock should report already held" ); - assert!(is_lock_failure, "unexpected error: {err}"); - repo.unlock(&id).unwrap(); + lock.unlock().unwrap(); } -#[test] -fn commit_failure_keeps_lock_until_abort() { - let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); +#[tokio::test] +async fn commit_failure_keeps_lock_until_abort() { + let repo = Arc::new( + HashMapRepository::new() + .queued_async() + .async_aggregate::(), + ); let mut todo = Todo::new(); let id = next_id(); todo.initialize( @@ -528,21 +572,24 @@ fn commit_failure_keeps_lock_until_abort() { "Commit failure lock".to_string(), ) .unwrap(); - repo.commit(&mut todo).unwrap(); - - let mut locked = repo.get(&id).unwrap().unwrap(); - let mut concurrent = repo - .repo() - .inner() - .get_aggregate::(&id) - .unwrap() - .unwrap(); + repo.commit(&mut todo).await.unwrap(); + + let mut locked = repo.get(&id).await.unwrap().unwrap(); + + // Simulate a concurrent writer that bumps the version out from under the + // locked aggregate. The inner repository is unlocked (the queue lock lives in + // the `QueuedRepository` wrapper held by the main thread), and a cheap clone + // shares the same `Arc`-backed store, so this writes to the same namespaced + // stream without contending on the lock. + let inner = repo.repo().inner().clone().async_aggregate::(); + let mut concurrent = inner.get(&id).await.unwrap().unwrap(); concurrent.complete().unwrap(); - repo.repo().inner().commit(&mut concurrent.entity).unwrap(); + inner.commit(&mut concurrent).await.unwrap(); locked.complete().unwrap(); let err = repo .commit(&mut locked) + .await .expect_err("stale locked aggregate should fail optimistic commit"); assert!( matches!(err, RepositoryError::ConcurrentWrite { .. }), @@ -555,7 +602,8 @@ fn commit_failure_keeps_lock_until_abort() { let id_other = id.clone(); thread::spawn(move || { tx_started.send(()).unwrap(); - let todo = repo_other.get(&id_other).unwrap().unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let todo = rt.block_on(repo_other.get(&id_other)).unwrap().unwrap(); repo_other.abort(&todo).unwrap(); tx_got.send(()).unwrap(); }); @@ -567,8 +615,8 @@ fn commit_failure_keeps_lock_until_abort() { assert!(rx_got.recv_timeout(Duration::from_millis(500)).is_ok()); } -#[test] -fn outbox_worker_process_next_with_commit() { +#[tokio::test] +async fn outbox_worker_process_next_with_commit() { let repo = HashMapRepository::new(); let mut todo = Todo::new(); let id = next_id(); @@ -591,10 +639,11 @@ fn outbox_worker_process_next_with_commit() { message3.id().to_string(), ]; - repo.outbox_sync(message1) - .outbox_sync(message2) - .outbox_sync(message3) - .commit_sync(&mut todo) + repo.outbox(message1) + .outbox(message2) + .outbox(message3) + .commit(&mut todo) + .await .unwrap(); let buffer = Arc::new(Mutex::new(Vec::new())); @@ -641,8 +690,8 @@ fn outbox_worker_process_next_with_commit() { } /// Full metadata chain: Entity → EventRecord → OutboxMessage → OutboxWorker → publisher -#[test] -fn metadata_flows_from_entity_through_outbox_to_publisher() { +#[tokio::test] +async fn metadata_flows_from_entity_through_outbox_to_publisher() { let repo = HashMapRepository::new(); // 1. Create a todo with metadata on the entity @@ -673,8 +722,8 @@ fn metadata_flows_from_entity_through_outbox_to_publisher() { assert_eq!(message.causation_id(), Some("cmd-create-todo")); // 4. Commit both using outbox commit builder - let repo = repo.aggregate::(); - repo.outbox_sync(message).commit_sync(&mut todo).unwrap(); + let repo = repo.async_aggregate::(); + repo.outbox(message).commit(&mut todo).await.unwrap(); // 5. Process through outbox worker, verify metadata reaches publisher let buffer = Arc::new(Mutex::new(Vec::new())); diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs index 7cd25357b..fa0264891 100644 --- a/tests/transport_conformance/mod.rs +++ b/tests/transport_conformance/mod.rs @@ -20,7 +20,7 @@ use sourced_rust::microsvc::transport::{ run_source, AsyncMessagePublisher, AsyncMessageSource, FailurePolicy, OutboxDispatcher, ReceivedMessage, RunOptions, TransportError, }; -use sourced_rust::microsvc::{HandlerError, Message, MessageKind, Service}; +use sourced_rust::microsvc::{Context, HandlerError, Message, MessageKind, Service}; use sourced_rust::{ CommitBatch, HashMapOutboxStore, HashMapRepository, OutboxMessage, OutboxMessageStatus, TransactionalCommit, @@ -183,19 +183,19 @@ pub fn recording_service(recorder: &Arc) -> Arc> { Arc::new( Service::new(()) .event("ok") - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { ok.push(Event::Handled(ctx.message().name().to_string())); - Ok(json!({})) + async move { Ok(json!({})) } }) .event("retryable") - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { retryable.push(Event::Handled(ctx.message().name().to_string())); - Err(HandlerError::Other("infra".into())) + async move { Err(HandlerError::Other("infra".into())) } }) .event("permanent") - .handle(move |ctx| { + .handle(move |ctx: &Context<()>| { permanent.push(Event::Handled(ctx.message().name().to_string())); - Err(HandlerError::Rejected("nope".into())) + async move { Err(HandlerError::Rejected("nope".into())) } }), ) } diff --git a/tests/upcasting/aggregate.rs b/tests/upcasting/aggregate.rs index a02adf93f..473004567 100644 --- a/tests/upcasting/aggregate.rs +++ b/tests/upcasting/aggregate.rs @@ -32,7 +32,7 @@ impl TodoV1 { } } -sourced_rust::aggregate!(TodoV1, entity { +sourced_rust::aggregate!(TodoV1, entity, aggregate_type = "Todo" { "Initialized"(id, user_id, task) => initialize, "Completed"() => complete(), }); @@ -70,7 +70,7 @@ impl TodoV2 { } } -sourced_rust::aggregate!(TodoV2, entity { +sourced_rust::aggregate!(TodoV2, entity, aggregate_type = "Todo" { "Initialized"(id, user_id, task, priority) => initialize, "Completed"() => complete(), } upcasters [ @@ -119,7 +119,7 @@ impl TodoV3 { } } -sourced_rust::aggregate!(TodoV3, entity { +sourced_rust::aggregate!(TodoV3, entity, aggregate_type = "Todo" { "Initialized"(id, user_id, task, priority, due_date) => initialize, "Completed"() => complete(), } upcasters [ diff --git a/tests/upcasting/main.rs b/tests/upcasting/main.rs index 196c4ed42..944c42e55 100644 --- a/tests/upcasting/main.rs +++ b/tests/upcasting/main.rs @@ -2,9 +2,9 @@ mod aggregate; use aggregate::{TodoV1, TodoV2, TodoV3}; use sourced_rust::{ - hydrate, hydrate_from_snapshot, upcast_events, Aggregate, AggregateBuilder, Commit, Entity, - EventRecord, EventUpcaster, HashMapRepository, RepositoryError, SnapshotRecord, SnapshotStore, - UpcastError, + hydrate, hydrate_from_snapshot, upcast_events, Aggregate, AsyncAggregateBuilder, + AsyncSnapshotStore, Entity, EventRecord, EventUpcaster, HashMapRepository, RepositoryError, + SnapshotRecord, StreamIdentity, UpcastError, }; fn identity_payload(event: &EventRecord) -> Result, UpcastError> { @@ -260,18 +260,23 @@ fn mixed_events_v1_init_and_v1_complete() { // Repository round-trip with upcasting // ============================================================================= -#[test] -fn repo_roundtrip_v1_to_v2() { +#[tokio::test] +async fn repo_roundtrip_v1_to_v2() { // Store using v1 - let v1_repo = HashMapRepository::new(); + let base_repo = HashMapRepository::new(); let mut v1 = TodoV1::default(); v1.initialize("t1".into(), "frank".into(), "Shop".into()) .unwrap(); - v1_repo.commit(&mut v1.entity).unwrap(); + base_repo + .clone() + .async_aggregate::() + .commit(&mut v1) + .await + .unwrap(); // Load using v2 (same storage) - let v2_repo = v1_repo.aggregate::(); - let loaded = v2_repo.get("t1").unwrap().unwrap(); + let v2_repo = base_repo.async_aggregate::(); + let loaded = v2_repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.user_id, "frank"); assert_eq!(loaded.task, "Shop"); assert_eq!(loaded.priority, 0); @@ -331,34 +336,41 @@ fn hydrate_returns_replay_error_when_typed_upcaster_decode_fails() { // Snapshot + upcasting // ============================================================================= -#[test] -fn snapshot_plus_upcasting_post_snapshot_events() { +#[tokio::test] +async fn snapshot_plus_upcasting_post_snapshot_events() { // TodoV2 implements Snapshottable in aggregate.rs let repo = HashMapRepository::new() - .aggregate::() + .async_aggregate::() .with_snapshots(1); // Create using native v2 with a specific priority let mut todo = TodoV2::default(); todo.initialize("t1".into(), "grace".into(), "Run".into(), 7) .unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); // Snapshot should now exist at version 1 - assert!(repo.repo().repo().get_snapshot("t1").unwrap().is_some()); + let snapshot_identity = StreamIdentity::new(TodoV2::aggregate_type(), "t1").unwrap(); + assert!(repo + .repo() + .repo() + .get_snapshot_async(&snapshot_identity) + .await + .unwrap() + .is_some()); // Add another event; this triggers snapshot + partial replay path - let mut todo = repo.get("t1").unwrap().unwrap(); + let mut todo = repo.get("t1").await.unwrap().unwrap(); todo.complete().unwrap(); - repo.commit(&mut todo).unwrap(); + repo.commit(&mut todo).await.unwrap(); - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.priority, 7); assert!(loaded.completed); } -#[test] -fn snapshot_repo_with_v1_events_upcasted_on_hydrate() { +#[tokio::test] +async fn snapshot_repo_with_v1_events_upcasted_on_hydrate() { // Store v1 events, then load with v2 snapshot repo let base_repo = HashMapRepository::new(); @@ -367,12 +379,17 @@ fn snapshot_repo_with_v1_events_upcasted_on_hydrate() { v1.initialize("t1".into(), "hank".into(), "Sweep".into()) .unwrap(); v1.complete().unwrap(); - base_repo.commit(&mut v1.entity).unwrap(); + base_repo + .clone() + .async_aggregate::() + .commit(&mut v1) + .await + .unwrap(); // Load via a v2 snapshot-aware repo (no snapshot exists, so full replay with upcasting) - let repo = base_repo.aggregate::().with_snapshots(5); + let repo = base_repo.async_aggregate::().with_snapshots(5); - let loaded = repo.get("t1").unwrap().unwrap(); + let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.user_id, "hank"); assert_eq!(loaded.task, "Sweep"); assert_eq!(loaded.priority, 0); // upcasted default From 5a67159c0e5e8ca4e78eaa2fabde27493d855fa4 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 20:45:24 -0500 Subject: [PATCH 37/39] =?UTF-8?q?refactor!:=20remove=20the=20sync=20reposi?= =?UTF-8?q?tory=20API=20=E2=80=94=20the=20crate=20is=20now=20async-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes the entire synchronous repository/read-model/snapshot trait surface, now unused after the async handler switch. This eliminates the sync/async mix that was the source of subtle combination bugs: there is exactly one (async) path for every backend. Removed (traits + all backend impls + re-exports): - repository: Get/Commit/Repository (repository.rs), GetOne/GetMany/Gettable (gettable.rs), the TransactionalCommit trait (batch.rs; CommitBatch kept). - snapshot: sync SnapshotStore + sync SnapshotAggregateRepository/SnapshotOutboxCommit. - read_model: sync ReadModelWritePlanStore/RelationalReadModelQueryStore, the sync ReadModelWorkspace load/commit impl, ReadModelLoadBuilder, ReadModelWorkspaceExt, and ReadModelWritePlanBuilder::commit (async equivalents kept). - aggregate: GetAggregate/GetAllAggregates/CommitAggregate + the sync AggregateRepository/AggregateBuilder (AsyncAggregateRepository/Builder kept). - commit_builder: SyncCommitBuilder/SyncStagedCommitBuilder/exts. - outbox: SyncOutboxCommit/SyncOutboxCommitExt (outbox_sync/commit_sync). - hashmap/postgres/sqlite/in-memory backends: their sync impls. - queued_repo: the sync QueuedRepository impls + sync Queueable::queued; the sync lock module (Lock/LockManager/InMemoryLock/InMemoryLockManager) is now fully unused and deleted (Async lock variants kept; LockError kept). - src/ unit tests that exercised the removed sync surface, converted to async. Also converted 5 remaining fully-sync integration crates the earlier sweep missed (bomberman [19 files], read_model_relationship_includes, read_model_commit_bridge, sourced_upcasting, transport_conformance's store_outbox). cargo test: 490 passed / 0 failed; --features http,grpc / postgres / sqlite all build; clippy clean; no sync trait remains in src/. Completes Phase 3 of [[tasks/async-only-consolidation]] — HashMap/SQLite/Postgres all async-only and consistent. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/aggregate/aggregate.rs | 215 +------ src/aggregate/mod.rs | 5 +- src/commit_builder/mod.rs | 551 +++++------------- src/hashmap_repo/repository.rs | 280 ++------- src/lib.rs | 36 +- src/lock/in_memory.rs | 182 ------ src/lock/lock.rs | 18 - src/lock/lock_manager.rs | 20 - src/lock/mod.rs | 6 - src/microsvc/transport/outbox_dispatch.rs | 6 +- src/microsvc/transport/outbox_source.rs | 8 +- src/outbox/commit.rs | 129 ++-- src/outbox/mod.rs | 11 +- src/outbox_worker/store.rs | 26 +- src/queued_repo/mod.rs | 4 +- src/queued_repo/repository.rs | 213 +------ src/read_model/in_memory.rs | 246 ++++---- src/read_model/mod.rs | 11 +- src/read_model/session.rs | 116 +--- src/repository/batch.rs | 6 - src/repository/gettable.rs | 150 ----- src/repository/mod.rs | 6 +- src/repository/repository.rs | 28 - src/snapshot/in_memory.rs | 139 ++--- src/snapshot/mod.rs | 6 +- src/snapshot/repository.rs | 300 ++-------- src/snapshot/store.rs | 15 - tests/bomberman/handlers/create_game.rs | 14 +- tests/bomberman/handlers/get_player.rs | 8 +- tests/bomberman/handlers/join_game.rs | 30 +- tests/bomberman/handlers/mod.rs | 1 + tests/bomberman/handlers/move_player.rs | 37 +- tests/bomberman/handlers/place_bomb.rs | 38 +- tests/bomberman/handlers/shared.rs | 49 +- tests/bomberman/handlers/tick.rs | 64 +- tests/bomberman/main.rs | 369 ++++++------ tests/bomberman/sim.rs | 49 +- tests/read_model_commit_bridge/main.rs | 18 +- .../read_model_relationship_includes/main.rs | 391 +++++++------ tests/sourced_upcasting/main.rs | 23 +- tests/transport_conformance/mod.rs | 18 +- 41 files changed, 1194 insertions(+), 2648 deletions(-) delete mode 100644 src/lock/in_memory.rs delete mode 100644 src/lock/lock.rs delete mode 100644 src/lock/lock_manager.rs delete mode 100644 src/repository/gettable.rs delete mode 100644 src/repository/repository.rs diff --git a/src/aggregate/aggregate.rs b/src/aggregate/aggregate.rs index d1016fccd..0031e82cc 100644 --- a/src/aggregate/aggregate.rs +++ b/src/aggregate/aggregate.rs @@ -1,12 +1,7 @@ use std::fmt; -use std::marker::PhantomData; use crate::entity::{upcast_events, Entity, EventRecord, EventUpcaster}; -use crate::queued_repo::{GetAllWithOpts, GetWithOpts, ReadOpts, UnlockableRepository}; -use crate::repository::{ - Commit, CommitBatch, Get, Repository, RepositoryError, TransactionalCommit, -}; -use crate::snapshot::{SnapshotAggregateRepository, SnapshotStore, Snapshottable}; +use crate::repository::RepositoryError; /// Trait for domain aggregates that can be event-sourced. pub trait Aggregate: Sized + Default { @@ -123,211 +118,3 @@ pub fn hydrate(entity: Entity) -> Result { Ok(agg) } - -/// Extension trait adding aggregate-aware get method. -pub trait GetAggregate: Get { - fn get_aggregate(&self, id: &str) -> Result, RepositoryError> - where - Self: Sized, - { - let entity = self.get(id)?; - let Some(entity) = entity else { - return Ok(None); - }; - Ok(Some(hydrate::(entity)?)) - } -} - -impl GetAggregate for R {} - -/// Extension trait adding aggregate-aware get_all method. -pub trait GetAllAggregates: Get { - fn get_all_aggregates(&self, ids: &[&str]) -> Result, RepositoryError> - where - Self: Sized, - { - let entities = self.get(ids)?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - aggregates.push(hydrate::(entity)?); - } - Ok(aggregates) - } -} - -impl GetAllAggregates for R {} - -/// Extension trait adding aggregate-aware commit methods. -pub trait CommitAggregate: Commit { - fn commit_aggregate(&self, aggregate: &mut A) -> Result<(), RepositoryError> { - self.commit(aggregate.entity_mut()) - } - - fn commit_all_aggregates( - &self, - aggregates: &mut [&mut A], - ) -> Result<(), RepositoryError> - where - Self: TransactionalCommit, - { - let entities: Vec<&mut Entity> = aggregates - .iter_mut() - .map(|agg| (*agg).entity_mut()) - .collect(); - self.commit_batch(CommitBatch::new(entities)) - } -} - -impl CommitAggregate for R {} - -/// Combined extension trait for full repository aggregate support. -pub trait RepositoryExt: GetAggregate + GetAllAggregates + CommitAggregate {} - -impl RepositoryExt for R {} - -/// Builder trait for creating typed aggregate repositories. -pub trait AggregateBuilder: Sized { - fn aggregate(self) -> AggregateRepository { - AggregateRepository::new(self) - } -} - -impl AggregateBuilder for T {} - -/// A repository wrapper that provides typed access to a specific aggregate type. -pub struct AggregateRepository { - repo: R, - _marker: PhantomData, -} - -impl AggregateRepository { - pub fn new(repo: R) -> Self { - AggregateRepository { - repo, - _marker: PhantomData, - } - } - - pub fn repo(&self) -> &R { - &self.repo - } - - pub fn repo_mut(&mut self) -> &mut R { - &mut self.repo - } -} - -impl AggregateRepository -where - R: Get, - A: Aggregate, -{ - pub fn get(&self, id: &str) -> Result, RepositoryError> { - let entity = self.repo.get(id)?; - let Some(entity) = entity else { - return Ok(None); - }; - Ok(Some(hydrate::(entity)?)) - } -} - -impl AggregateRepository -where - R: Get, - A: Aggregate, -{ - pub fn get_all(&self, ids: &[&str]) -> Result, RepositoryError> { - let entities = self.repo.get(ids)?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - aggregates.push(hydrate::(entity)?); - } - Ok(aggregates) - } -} - -impl AggregateRepository -where - R: Commit, - A: Aggregate, -{ - pub fn commit(&self, aggregate: &mut A) -> Result<(), RepositoryError> { - self.repo.commit(aggregate.entity_mut()) - } -} - -impl AggregateRepository -where - R: TransactionalCommit, - A: Aggregate, -{ - pub fn commit_all(&self, aggregates: &mut [&mut A]) -> Result<(), RepositoryError> { - let entities: Vec<&mut Entity> = aggregates - .iter_mut() - .map(|agg| (*agg).entity_mut()) - .collect(); - self.repo.commit_batch(CommitBatch::new(entities)) - } -} - -impl AggregateRepository -where - R: UnlockableRepository, - A: Aggregate, -{ - pub fn abort(&self, aggregate: &A) -> Result<(), RepositoryError> { - self.repo.unlock(aggregate.entity().id()) - } -} - -impl AggregateRepository -where - R: SnapshotStore, - A: Snapshottable, -{ - /// Wrap this repository with snapshot support at the given event frequency. - pub fn with_snapshots(self, frequency: u64) -> SnapshotAggregateRepository { - SnapshotAggregateRepository::new(self, frequency) - } -} - -impl AggregateRepository -where - R: GetWithOpts, - A: Aggregate, -{ - /// Get an aggregate with options (e.g., to skip locking). - pub fn get_with(&self, id: &str, opts: ReadOpts) -> Result, RepositoryError> { - let entity = self.repo.get_with(id, opts)?; - let Some(entity) = entity else { - return Ok(None); - }; - Ok(Some(hydrate::(entity)?)) - } - - /// Non-locking read (alias for get_with no_lock). - pub fn peek(&self, id: &str) -> Result, RepositoryError> { - self.get_with(id, ReadOpts::no_lock()) - } -} - -impl AggregateRepository -where - R: GetAllWithOpts, - A: Aggregate, -{ - /// Get all aggregates with options (e.g., to skip locking). - pub fn get_all_with(&self, ids: &[&str], opts: ReadOpts) -> Result, RepositoryError> { - let entities = self.repo.get_all_with(ids, opts)?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - aggregates.push(hydrate::(entity)?); - } - Ok(aggregates) - } - - /// Non-locking read (alias for get_all_with no_lock). - pub fn peek_all(&self, ids: &[&str]) -> Result, RepositoryError> { - self.get_all_with(ids, ReadOpts::no_lock()) - } -} diff --git a/src/aggregate/mod.rs b/src/aggregate/mod.rs index a5a0c7c60..137e11174 100644 --- a/src/aggregate/mod.rs +++ b/src/aggregate/mod.rs @@ -1,8 +1,5 @@ mod aggregate; mod async_aggregate; -pub use aggregate::{ - hydrate, Aggregate, AggregateBuilder, AggregateRepository, CommitAggregate, GetAggregate, - GetAllAggregates, RepositoryExt, -}; +pub use aggregate::{hydrate, Aggregate}; pub use async_aggregate::{AsyncAggregateBuilder, AsyncAggregateRepository}; diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index 21febbb89..f5a42667a 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -1,5 +1,5 @@ -//! SyncCommitBuilder and AsyncCommitBuilder chain read models, write plans, -//! outbox, and aggregates into one transactional batch. +//! AsyncCommitBuilder chains read models, write plans, outbox, and aggregates +//! into one transactional batch. //! //! ## Example //! @@ -9,35 +9,6 @@ //! read_models.upsert_related(&player, "weapons", &weapon)?; //! //! repo -//! .read_models_sync(read_models) -//! .commit_sync(&mut game)?; -//! -//! // Ordering is semantic staging only. -//! let mut read_models = sourced_rust::ReadModelWritePlanBuilder::new(); -//! read_models.upsert(&player)?; -//! read_models.upsert_related(&player, "weapons", &weapon)?; -//! -//! repo -//! .outbox_sync(message) -//! .read_models_sync(read_models) -//! .commit_sync(&mut game)?; -//! -//! let mut read_models = sourced_rust::ReadModelWritePlanBuilder::new(); -//! read_models.upsert(&player)?; -//! read_models.upsert_related(&player, "weapons", &weapon)?; -//! -//! repo -//! .aggregate_sync(&mut game) -//! .read_models_sync(read_models) -//! .outbox_sync(message) -//! .commit_sync()?; -//! -//! // Async repositories use the same staging shape. -//! let mut read_models = sourced_rust::ReadModelWritePlanBuilder::new(); -//! read_models.upsert(&player)?; -//! read_models.upsert_related(&player, "weapons", &weapon)?; -//! -//! repo //! .read_models(read_models) //! .commit(&mut game) //! .await?; @@ -48,19 +19,9 @@ use crate::entity::Entity; use crate::outbox::OutboxMessage; use crate::read_model::{ReadModelWritePlan, ReadModelWritePlanBuilder}; use crate::repository::{ - AsyncCommitBatch, AsyncStreamWrite, AsyncTransactionalCommit, CommitBatch, RepositoryError, - StreamIdentity, TransactionalCommit, + AsyncCommitBatch, AsyncStreamWrite, AsyncTransactionalCommit, RepositoryError, StreamIdentity, }; -/// Builder for chaining multiple items into a single transactional commit batch. -pub struct SyncCommitBuilder<'a, R> { - repo: &'a R, - entities: Vec, - outbox_messages: Vec, - read_model_plans: Vec, - error: Option, -} - #[derive(Clone, Debug, PartialEq, Eq)] struct OutboxSource { aggregate_type: String, @@ -113,233 +74,6 @@ impl StagedOutboxSource { } } -impl<'a, R> SyncCommitBuilder<'a, R> { - pub fn new(repo: &'a R) -> Self { - Self { - repo, - entities: vec![], - outbox_messages: vec![], - read_model_plans: vec![], - error: None, - } - } - - /// Add a read-model write plan builder to the commit. - pub fn read_models_sync(mut self, read_models: ReadModelWritePlanBuilder) -> Self { - if self.error.is_some() { - return self; - } - - match read_models.into_write_plan() { - Ok(plan) => self.read_model_plans.push(plan), - Err(err) => self.error = Some(err.into()), - } - self - } - - /// Add an outbox message to the commit (takes ownership). - pub fn outbox_sync(mut self, msg: OutboxMessage) -> Self { - self.outbox_messages.push(msg); - self - } - - /// Stage an aggregate and switch to a no-argument staged commit builder. - pub fn aggregate_sync( - self, - aggregate: &'a mut A, - ) -> SyncStagedCommitBuilder<'a, R> { - let source = OutboxSource::from_aggregate(aggregate); - let mut builder = SyncStagedCommitBuilder::from_builder(self); - builder.outbox_source.record(source); - builder.staged_entities.push(aggregate.entity_mut()); - builder - } - - /// Commit all items plus the primary aggregate. - pub fn commit_sync(mut self, aggregate: &mut A) -> Result<(), RepositoryError> - where - R: TransactionalCommit, - { - self.check_staged()?; - for message in &mut self.outbox_messages { - message.set_source(aggregate); - } - - let mut entity_refs: Vec<&mut Entity> = self.entities.iter_mut().collect(); - entity_refs.push(aggregate.entity_mut()); - self.repo.commit_batch(CommitBatch { - entities: entity_refs, - outbox_messages: self.outbox_messages, - read_model_plans: self.read_model_plans, - snapshots: Vec::new(), - inbox_receipts: Vec::new(), - }) - } - - /// Commit multiple entities in one batch (along with any queued read-model plans and outbox). - /// - /// Use `entity_mut()` on each aggregate to get the entity references: - /// ```ignore - /// repo.read_models_sync(read_models) - /// .commit_many_sync(&mut [player.entity_mut(), monster.entity_mut()])?; - /// ``` - pub fn commit_many_sync(mut self, entities: &mut [&mut Entity]) -> Result<(), RepositoryError> - where - R: TransactionalCommit, - { - self.check_staged()?; - - let mut entity_refs: Vec<&mut Entity> = self.entities.iter_mut().collect(); - for e in entities.iter_mut() { - entity_refs.push(&mut **e); - } - self.repo.commit_batch(CommitBatch { - entities: entity_refs, - outbox_messages: self.outbox_messages, - read_model_plans: self.read_model_plans, - snapshots: Vec::new(), - inbox_receipts: Vec::new(), - }) - } - - /// Commit without a primary aggregate. - pub fn commit_all_sync(mut self) -> Result<(), RepositoryError> - where - R: TransactionalCommit, - { - self.check_staged()?; - - let entity_refs: Vec<&mut Entity> = self.entities.iter_mut().collect(); - self.repo.commit_batch(CommitBatch { - entities: entity_refs, - outbox_messages: self.outbox_messages, - read_model_plans: self.read_model_plans, - snapshots: Vec::new(), - inbox_receipts: Vec::new(), - }) - } - - fn check_staged(&mut self) -> Result<(), RepositoryError> { - if let Some(err) = self.error.take() { - return Err(err); - } - Ok(()) - } -} - -/// Builder returned after one or more aggregates are staged explicitly. -pub struct SyncStagedCommitBuilder<'a, R> { - repo: &'a R, - entities: Vec, - outbox_messages: Vec, - staged_entities: Vec<&'a mut Entity>, - outbox_source: StagedOutboxSource, - read_model_plans: Vec, - error: Option, -} - -impl<'a, R> SyncStagedCommitBuilder<'a, R> { - fn from_builder(builder: SyncCommitBuilder<'a, R>) -> Self { - Self { - repo: builder.repo, - entities: builder.entities, - outbox_messages: builder.outbox_messages, - staged_entities: Vec::new(), - outbox_source: StagedOutboxSource::default(), - read_model_plans: builder.read_model_plans, - error: builder.error, - } - } - - pub fn read_models_sync(mut self, read_models: ReadModelWritePlanBuilder) -> Self { - if self.error.is_some() { - return self; - } - - match read_models.into_write_plan() { - Ok(plan) => self.read_model_plans.push(plan), - Err(err) => self.error = Some(err.into()), - } - self - } - - pub fn outbox_sync(mut self, msg: OutboxMessage) -> Self { - self.outbox_messages.push(msg); - self - } - - pub fn aggregate_sync(mut self, aggregate: &'a mut A) -> Self { - self.outbox_source - .record(OutboxSource::from_aggregate(aggregate)); - self.staged_entities.push(aggregate.entity_mut()); - self - } - - pub fn entity_sync(mut self, entity: &'a mut Entity) -> Self { - self.staged_entities.push(entity); - self - } - - pub fn commit_sync(mut self) -> Result<(), RepositoryError> - where - R: TransactionalCommit, - { - self.check_staged()?; - self.outbox_source.apply_to(&mut self.outbox_messages); - - let mut entity_refs: Vec<&mut Entity> = self.entities.iter_mut().collect(); - entity_refs.extend(self.staged_entities); - self.repo.commit_batch(CommitBatch { - entities: entity_refs, - outbox_messages: self.outbox_messages, - read_model_plans: self.read_model_plans, - snapshots: Vec::new(), - inbox_receipts: Vec::new(), - }) - } - - fn check_staged(&mut self) -> Result<(), RepositoryError> { - if let Some(err) = self.error.take() { - return Err(err); - } - Ok(()) - } -} - -/// Extension trait to start a commit builder chain from an outbox message. -pub trait SyncCommitBuilderExt: TransactionalCommit + Sized { - /// Start a commit builder chain with an outbox message. - fn outbox_sync(&self, msg: OutboxMessage) -> SyncCommitBuilder<'_, Self> { - SyncCommitBuilder::new(self).outbox_sync(msg) - } -} - -impl SyncCommitBuilderExt for R {} - -/// Extension trait for relational read-model write-plan commit entrypoints. -/// -/// Kept separate from `SyncCommitBuilderExt` so callers explicitly opt into the -/// write-plan starter. -pub trait SyncReadModelWritePlanCommitExt: TransactionalCommit + Sized { - /// Start a commit builder chain with a relational read-model write plan. - fn read_models_sync( - &self, - read_models: ReadModelWritePlanBuilder, - ) -> SyncCommitBuilder<'_, Self> { - SyncCommitBuilder::new(self).read_models_sync(read_models) - } - - /// Start a staged commit builder with an aggregate. - fn aggregate_sync<'a, A: Aggregate>( - &'a self, - aggregate: &'a mut A, - ) -> SyncStagedCommitBuilder<'a, Self> { - SyncCommitBuilder::new(self).aggregate_sync(aggregate) - } -} - -impl SyncReadModelWritePlanCommitExt for R {} - /// Async builder for chaining multiple items into one transactional commit batch. pub struct AsyncCommitBuilder<'a, R> { repo: &'a R, @@ -582,13 +316,31 @@ impl AsyncReadModelWritePlanCommitExt for R {} mod tests { use super::*; use crate::{ - sourced, AsyncTransactionalCommit, Entity, Get, HashMapRepository, ReadModelWorkspaceExt, + sourced, AsyncReadModelWorkspaceExt, AsyncTransactionalCommit, Entity, HashMapRepository, RowKey, RowValue, }; use serde::{Deserialize, Serialize}; - use std::cell::RefCell; use std::sync::Mutex; + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + type OutboxSourceRecord = (String, Option, Option, Option); #[derive(Default)] @@ -614,61 +366,6 @@ mod tests { counter: i32, } - #[derive(Default)] - struct RecordingBatchRepo { - fail: bool, - entity_ids: RefCell>, - outbox_ids: RefCell>, - outbox_sources: RefCell>, - read_model_keys: RefCell>, - } - - impl TransactionalCommit for RecordingBatchRepo { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - *self.entity_ids.borrow_mut() = batch - .entities - .iter() - .map(|entity| entity.id().to_string()) - .collect(); - *self.outbox_ids.borrow_mut() = batch - .outbox_messages - .iter() - .map(|message| message.id().to_string()) - .collect(); - *self.outbox_sources.borrow_mut() = batch - .outbox_messages - .iter() - .map(|message| { - ( - message.id().to_string(), - message.source_aggregate_type.clone(), - message.source_aggregate_id.clone(), - message.source_sequence, - ) - }) - .collect(); - *self.read_model_keys.borrow_mut() = batch - .read_model_plans - .iter() - .flat_map(|plan| { - plan.mutations - .iter() - .map(|mutation| mutation.lock_key()) - .collect::>() - }) - .collect(); - - if self.fail { - return Err(RepositoryError::Model("injected batch failure".into())); - } - - for entity in batch.entities { - entity.mark_committed(); - } - Ok(()) - } - } - #[derive(Default)] struct RecordingAsyncBatchRepo { fail: bool, @@ -756,12 +453,14 @@ mod tests { } fn loaded_view(repo: &HashMapRepository, id: &str) -> Option { - repo.model_store() - .workspace() - .load::(view_key(id)) - .one() - .unwrap() - .map(|versioned| versioned.data) + block_on( + repo.model_store() + .workspace_async() + .load_async::(view_key(id)) + .one(), + ) + .unwrap() + .map(|versioned| versioned.data) } #[test] @@ -776,9 +475,11 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .commit_sync(&mut agg) - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .commit(&mut agg), + ) + .unwrap(); let loaded = loaded_view(&repo, "1").unwrap(); assert_eq!(loaded.counter, 42); @@ -804,9 +505,10 @@ mod tests { let mut read_models = crate::read_model::ReadModelWritePlanBuilder::new(); read_models.upsert(&view1).unwrap().upsert(&view2).unwrap(); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models) - .commit_sync(&mut agg) - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models).commit(&mut agg), + ) + .unwrap(); assert_eq!(loaded_view(&repo, "1").unwrap().counter, 10); assert_eq!(loaded_view(&repo, "2").unwrap().counter, 20); @@ -826,10 +528,12 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .outbox_sync(outbox) - .commit_sync(&mut agg) - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .commit(&mut agg), + ) + .unwrap(); assert_eq!(loaded_view(&repo, "1").unwrap().counter, 42); } @@ -848,10 +552,12 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - SyncCommitBuilderExt::outbox_sync(&repo, outbox) - .read_models_sync(read_models(&view)) - .commit_sync(&mut agg) - .unwrap(); + block_on( + AsyncCommitBuilderExt::outbox(&repo, outbox) + .read_models(read_models(&view)) + .commit(&mut agg), + ) + .unwrap(); assert_eq!(loaded_view(&repo, "1").unwrap().counter, 99); } @@ -872,8 +578,7 @@ mod tests { let mut read_models = crate::read_model::ReadModelWritePlanBuilder::new(); read_models.upsert(&view1).unwrap().upsert(&view2).unwrap(); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models) - .commit_all_sync() + block_on(AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models).commit_all()) .unwrap(); assert_eq!( @@ -903,22 +608,33 @@ mod tests { agg2.touch().unwrap(); agg2.entity.set_id("agg-2"); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .commit_many_sync(&mut [agg1.entity_mut(), agg2.entity_mut()]) - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .commit_many(&mut [&mut agg1, &mut agg2]), + ) + .unwrap(); assert_eq!(loaded_view(&repo, "multi").unwrap().counter, 77); - let e1 = repo.get("agg-1").unwrap(); + let agg_type = TestAggregate::aggregate_type(); + let e1 = block_on(crate::AsyncGetStream::get_stream( + &repo, + &StreamIdentity::new(agg_type, "agg-1").unwrap(), + )) + .unwrap(); assert!(e1.is_some()); - let e2 = repo.get("agg-2").unwrap(); + let e2 = block_on(crate::AsyncGetStream::get_stream( + &repo, + &StreamIdentity::new(agg_type, "agg-2").unwrap(), + )) + .unwrap(); assert!(e2.is_some()); } #[test] fn staged_builder_ordering_is_semantic_for_outbox_session_and_aggregate() { - fn record(order: u8) -> (Vec, Vec) { - let repo = RecordingBatchRepo::default(); + fn record(order: u8) -> (Vec<(String, String)>, Vec) { + let repo = RecordingAsyncBatchRepo::default(); let view = RelationalView { id: "ordered".into(), counter: 7, @@ -928,29 +644,32 @@ mod tests { agg.touch().unwrap(); match order { - 0 => SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .outbox_sync(outbox) - .aggregate_sync(&mut agg) - .commit_sync() - .unwrap(), - 1 => repo - .outbox_sync(outbox) - .read_models_sync(read_models(&view)) - .aggregate_sync(&mut agg) - .commit_sync() - .unwrap(), - _ => SyncReadModelWritePlanCommitExt::aggregate_sync(&repo, &mut agg) - .read_models_sync(read_models(&view)) - .outbox_sync(outbox) - .commit_sync() - .unwrap(), + 0 => block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .aggregate(&mut agg) + .commit(), + ) + .unwrap(), + 1 => block_on( + repo.outbox(outbox) + .read_models(read_models(&view)) + .aggregate(&mut agg) + .commit(), + ) + .unwrap(), + _ => block_on( + AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) + .read_models(read_models(&view)) + .outbox(outbox) + .commit(), + ) + .unwrap(), } - let recorded = ( - repo.entity_ids.borrow().clone(), - repo.read_model_keys.borrow().clone(), - ); - recorded + let stream_ids = repo.stream_ids.lock().unwrap().clone(); + let read_model_keys = repo.read_model_keys.lock().unwrap().clone(); + (stream_ids, read_model_keys) } let baseline = record(0); @@ -960,18 +679,20 @@ mod tests { #[test] fn staged_commit_sets_outbox_source_from_single_aggregate() { - let repo = RecordingBatchRepo::default(); + let repo = RecordingAsyncBatchRepo::default(); let mut agg = TestAggregate::default(); agg.touch().unwrap(); let outbox = OutboxMessage::create("sourced-msg", "TestEvent", b"{}".to_vec()).unwrap(); - SyncReadModelWritePlanCommitExt::aggregate_sync(&repo, &mut agg) - .outbox_sync(outbox) - .commit_sync() - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) + .outbox(outbox) + .commit(), + ) + .unwrap(); assert_eq!( - repo.outbox_sources.borrow().as_slice(), + repo.outbox_sources.lock().unwrap().as_slice(), &[( "sourced-msg".to_string(), Some(TestAggregate::aggregate_type().to_string()), @@ -983,7 +704,7 @@ mod tests { #[test] fn staged_builder_supports_multiple_aggregates() { - let repo = RecordingBatchRepo::default(); + let repo = RecordingAsyncBatchRepo::default(); let view = RelationalView { id: "staged-multi".into(), counter: 77, @@ -995,25 +716,36 @@ mod tests { agg2.touch().unwrap(); agg2.entity.set_id("agg-2"); - SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .aggregate_sync(&mut agg1) - .aggregate_sync(&mut agg2) - .commit_sync() - .unwrap(); + block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .aggregate(&mut agg1) + .aggregate(&mut agg2) + .commit(), + ) + .unwrap(); assert_eq!( - repo.read_model_keys.borrow().as_slice(), + repo.read_model_keys.lock().unwrap().as_slice(), &[lock_key_for(&view)] ); assert_eq!( - repo.entity_ids.borrow().as_slice(), - &["agg-1".to_string(), "agg-2".to_string()] + repo.stream_ids.lock().unwrap().as_slice(), + &[ + ( + TestAggregate::aggregate_type().to_string(), + "agg-1".to_string() + ), + ( + TestAggregate::aggregate_type().to_string(), + "agg-2".to_string() + ), + ] ); } #[test] fn commit_builder_failure_does_not_mark_aggregate_committed() { - let repo = RecordingBatchRepo { + let repo = RecordingAsyncBatchRepo { fail: true, ..Default::default() }; @@ -1026,34 +758,45 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - let err = SyncReadModelWritePlanCommitExt::read_models_sync(&repo, read_models(&view)) - .outbox_sync(outbox) - .commit_sync(&mut agg) - .unwrap_err(); + let err = block_on( + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .commit(&mut agg), + ) + .unwrap_err(); - assert_eq!(err, RepositoryError::Model("injected batch failure".into())); + assert_eq!( + err, + RepositoryError::Model("injected async batch failure".into()) + ); assert_eq!(agg.entity().committed_version(), 0); assert_eq!(agg.entity().new_events().len(), 1); assert_eq!( - repo.read_model_keys.borrow().as_slice(), + repo.read_model_keys.lock().unwrap().as_slice(), &[lock_key_for(&view)] ); - assert!(repo.entity_ids.borrow().iter().any(|id| id == "agg-1")); + assert!(repo + .stream_ids + .lock() + .unwrap() + .iter() + .any(|(_, id)| id == "agg-1")); assert!(repo .outbox_ids - .borrow() + .lock() + .unwrap() .iter() .any(|id| id == "msg-rollback")); } #[test] fn commit_builder_empty_batch_succeeds() { - let repo = RecordingBatchRepo::default(); + let repo = RecordingAsyncBatchRepo::default(); - SyncCommitBuilder::new(&repo).commit_all_sync().unwrap(); + block_on(AsyncCommitBuilder::new(&repo).commit_all()).unwrap(); - assert!(repo.entity_ids.borrow().is_empty()); - assert!(repo.read_model_keys.borrow().is_empty()); + assert!(repo.stream_ids.lock().unwrap().is_empty()); + assert!(repo.read_model_keys.lock().unwrap().is_empty()); } #[tokio::test] diff --git a/src/hashmap_repo/repository.rs b/src/hashmap_repo/repository.rs index f432b039b..cf5275f5e 100644 --- a/src/hashmap_repo/repository.rs +++ b/src/hashmap_repo/repository.rs @@ -8,23 +8,20 @@ use std::future::Future; use std::sync::{Arc, RwLock}; use crate::entity::{ - Committable, Entity, EventRecord, EventRecordError, BITCODE_PAYLOAD_CODEC, - BITCODE_PAYLOAD_CODEC_VERSION, + Entity, EventRecord, EventRecordError, BITCODE_PAYLOAD_CODEC, BITCODE_PAYLOAD_CODEC_VERSION, }; use crate::outbox::OutboxMessage; use crate::read_model::in_memory::apply_read_model_write_plan; use crate::read_model::{ InMemoryReadModelStore, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, - ReadModelWritePlanStore, RelationalReadModelQueryStore, }; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, - AsyncTransactionalCommit, Commit, CommitBatch, GetMany, GetOne, PreparedEventAppend, - RepositoryError, SnapshotWrite, StreamIdentity, TransactionalCommit, + AsyncTransactionalCommit, PreparedEventAppend, RepositoryError, StreamIdentity, }; -use crate::snapshot::{InMemorySnapshotStore, SnapshotRecord, SnapshotStore}; +use crate::snapshot::{InMemorySnapshotStore, SnapshotRecord}; /// In-memory repository implementation using HashMap. /// @@ -96,36 +93,6 @@ impl HashMapRepository { } } -impl GetOne for HashMapRepository { - fn get_one(&self, id: &str) -> Result, RepositoryError> { - let storage = self - .event_store - .read() - .map_err(|_| RepositoryError::LockPoisoned("read"))?; - - if let Some(events) = storage.get(id) { - let mut entity = Entity::new(); - entity.set_id(id); - entity.load_from_history(events.clone()); - Ok(Some(entity)) - } else { - Ok(None) - } - } -} - -impl GetMany for HashMapRepository { - fn get_many(&self, ids: &[&str]) -> Result, RepositoryError> { - let mut entities = Vec::with_capacity(ids.len()); - for id in ids { - if let Some(entity) = self.get_one(id)? { - entities.push(entity); - } - } - Ok(entities) - } -} - impl AsyncGetStream for HashMapRepository { fn get_stream<'a>( &'a self, @@ -164,13 +131,6 @@ impl AsyncGetStream for HashMapRepository { } } -impl Commit for HashMapRepository { - fn commit(&self, committable: &mut C) -> Result<(), RepositoryError> { - let entities = committable.entities_mut(); - TransactionalCommit::commit_batch(self, CommitBatch::new(entities)) - } -} - impl AsyncTransactionalCommit for HashMapRepository { fn commit_batch_async<'a>( &'a self, @@ -297,109 +257,6 @@ impl AsyncInboxStore for HashMapRepository { } } -impl TransactionalCommit for HashMapRepository { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - reject_duplicate_streams(&batch.entities)?; - reject_duplicate_outbox_messages(&batch.outbox_messages)?; - - let mut storage = self - .event_store - .write() - .map_err(|_| RepositoryError::LockPoisoned("write"))?; - let mut relational_rows = self - .model_store - .relational_rows - .write() - .map_err(|_| RepositoryError::LockPoisoned("read model write"))?; - let mut snapshot_storage = self - .snapshot_store - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("snapshot write"))?; - let mut outbox_storage = self - .outbox_store - .write() - .map_err(|_| RepositoryError::LockPoisoned("outbox write"))?; - let mut inbox_storage = self - .inbox_store - .write() - .map_err(|_| RepositoryError::LockPoisoned("inbox write"))?; - - let mut staged_events = storage.clone(); - let mut staged_rows = relational_rows.clone(); - let mut staged_snapshots = snapshot_storage.clone(); - let mut staged_outbox = outbox_storage.clone(); - let mut staged_inbox = inbox_storage.clone(); - - // Phase 1: Validate all stream versions before staging any writes. - for entity in &batch.entities { - let stored_len = stored_stream_version(staged_events.get(entity.id())); - if stored_len != entity.committed_version() { - return Err(RepositoryError::ConcurrentWrite { - id: entity.id().to_string(), - expected: entity.committed_version(), - actual: stored_len, - }); - } - } - - // Phase 2: Apply every write to staged maps only. - for entity in &batch.entities { - let new_events = entity.new_events().to_vec(); - let stored = staged_events - .entry(entity.id().to_string()) - .or_insert_with(Vec::new); - stored.extend(new_events); - } - - for plan in batch.read_model_plans { - apply_read_model_write_plan(plan, &mut staged_rows)?; - } - - for write in batch.snapshots { - match write { - SnapshotWrite::Save(record) => { - record.validate()?; - staged_snapshots.insert(record.aggregate_id.clone(), record); - } - } - } - - for message in batch.outbox_messages { - let id = message.id().to_string(); - if staged_outbox.contains_key(&id) { - return Err(RepositoryError::DuplicateOutboxMessageInBatch { id }); - } - staged_outbox.insert(id, message); - } - - // Inbox receipts gate effectively-once (see the async impl). - for receipt in batch.inbox_receipts { - receipt.validate()?; - let key = (receipt.consumer.clone(), receipt.message_id.clone()); - if !staged_inbox.insert(key) { - return Err(RepositoryError::DuplicateInboxReceipt { - consumer: receipt.consumer, - message_id: receipt.message_id, - }); - } - } - - // Phase 3: Publish staged state only after all validation and staging succeeds. - *storage = staged_events; - *relational_rows = staged_rows; - *snapshot_storage = staged_snapshots; - *outbox_storage = staged_outbox; - *inbox_storage = staged_inbox; - - for entity in batch.entities { - entity.mark_committed(); - } - - Ok(()) - } -} - fn reject_duplicate_async_streams(streams: &[AsyncStreamWrite<'_>]) -> Result<(), RepositoryError> { let mut seen = HashSet::with_capacity(streams.len()); for stream in streams { @@ -495,86 +352,35 @@ fn validate_snapshot_identity( record.validate_for_identity(identity) } -fn reject_duplicate_streams(entities: &[&mut Entity]) -> Result<(), RepositoryError> { - let mut seen = HashSet::with_capacity(entities.len()); - for entity in entities { - let id = entity.id(); - if !seen.insert(id.to_string()) { - return Err(RepositoryError::DuplicateStreamInBatch { id: id.to_string() }); - } - } - Ok(()) -} - fn stored_stream_version(events: Option<&Vec>) -> u64 { // A missing stream has committed version 0; the first appended event will // occupy sequence 1. events.map_or(0, |events| events.len() as u64) } -impl ReadModelWritePlanStore for HashMapRepository { - fn read_model_capabilities(&self) -> ReadModelAdapterCapabilities { - ReadModelWritePlanStore::read_model_capabilities(&self.model_store) - } - - fn commit_write_plan( - &self, - plan: ReadModelWritePlan, - ) -> Result { - ReadModelWritePlanStore::commit_write_plan(&self.model_store, plan) - } -} - impl AsyncReadModelWritePlanStore for HashMapRepository { fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { - ReadModelWritePlanStore::read_model_capabilities(self) + self.model_store.read_model_capabilities_async() } fn commit_write_plan_async( &self, plan: ReadModelWritePlan, ) -> impl Future> + Send + '_ { - async move { ReadModelWritePlanStore::commit_write_plan(self, plan) } - } -} - -impl RelationalReadModelQueryStore for HashMapRepository { - fn read_model_query_capabilities(&self) -> ReadModelQueryCapabilities { - RelationalReadModelQueryStore::read_model_query_capabilities(&self.model_store) - } - - fn load_graph( - &self, - request: ReadModelLoadRequest, - ) -> Result { - RelationalReadModelQueryStore::load_graph(&self.model_store, request) + self.model_store.commit_write_plan_async(plan) } } impl AsyncRelationalReadModelQueryStore for HashMapRepository { fn read_model_query_capabilities_async(&self) -> ReadModelQueryCapabilities { - RelationalReadModelQueryStore::read_model_query_capabilities(self) + self.model_store.read_model_query_capabilities_async() } fn load_graph_async( &self, request: ReadModelLoadRequest, ) -> impl Future> + Send + '_ { - async move { RelationalReadModelQueryStore::load_graph(self, request) } - } -} - -impl SnapshotStore for HashMapRepository { - fn get_snapshot(&self, id: &str) -> Result, RepositoryError> { - SnapshotStore::get_snapshot(&self.snapshot_store, id) - } - - fn save_snapshot(&self, record: SnapshotRecord) -> Result<(), RepositoryError> { - SnapshotStore::save_snapshot(&self.snapshot_store, record) - } - - fn delete_snapshot(&self, id: &str) -> Result { - SnapshotStore::delete_snapshot(&self.snapshot_store, id) + self.model_store.load_graph_async(request) } } @@ -628,7 +434,39 @@ impl AsyncSnapshotStore for HashMapRepository { #[cfg(test)] mod tests { use super::*; - use crate::repository::Get; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + fn identity(id: &str) -> StreamIdentity { + StreamIdentity::new("test.aggregate", id).unwrap() + } + + fn commit_one(repo: &HashMapRepository, entity: &mut Entity) -> Result<(), RepositoryError> { + let id = entity.id().to_string(); + block_on( + repo.commit_batch_async(AsyncCommitBatch::new(vec![AsyncStreamWrite::new( + identity(&id), + entity, + )])), + ) + } #[test] fn new() { @@ -644,9 +482,9 @@ mod tests { entity.digest("test_event", &("arg1", "arg2")).unwrap(); - repo.commit(&mut entity).unwrap(); + commit_one(&repo, &mut entity).unwrap(); - let fetched_entity = repo.get(id).unwrap().unwrap(); + let fetched_entity = block_on(repo.get_stream(&identity(id))).unwrap().unwrap(); assert_eq!(fetched_entity.id(), id); assert_eq!(fetched_entity.events(), entity.events()); } @@ -661,10 +499,14 @@ mod tests { let mut entity2 = Entity::with_id("id_2"); entity2.digest("event2", &"arg2").unwrap(); - // Commit multiple entities using array syntax - repo.commit(&mut [&mut entity1, &mut entity2]).unwrap(); + block_on(repo.commit_batch_async(AsyncCommitBatch::new(vec![ + AsyncStreamWrite::new(identity("id_1"), &mut entity1), + AsyncStreamWrite::new(identity("id_2"), &mut entity2), + ]))) + .unwrap(); - let all_entities: Vec = repo.get(&["id_1", "id_2"]).unwrap(); + let all_entities: Vec = + block_on(repo.get_streams(&[identity("id_1"), identity("id_2")])).unwrap(); assert_eq!(all_entities.len(), 2); } @@ -678,15 +520,21 @@ mod tests { let mut entity2 = Entity::with_id("same-id"); entity2.digest("event2", &"arg2").unwrap(); - let err = repo.commit(&mut [&mut entity1, &mut entity2]).unwrap_err(); + let err = block_on(repo.commit_batch_async(AsyncCommitBatch::new(vec![ + AsyncStreamWrite::new(identity("same-id"), &mut entity1), + AsyncStreamWrite::new(identity("same-id"), &mut entity2), + ]))) + .unwrap_err(); assert_eq!( err, RepositoryError::DuplicateStreamInBatch { - id: "same-id".into() + id: identity("same-id").to_string(), } ); - assert!(repo.get("same-id").unwrap().is_none()); + assert!(block_on(repo.get_stream(&identity("same-id"))) + .unwrap() + .is_none()); assert_eq!(entity1.committed_version(), 0); assert_eq!(entity2.committed_version(), 0); assert_eq!(entity1.new_events().len(), 1); @@ -698,17 +546,17 @@ mod tests { use crate::repository::InboxReceipt; let repo = HashMapRepository::new(); - let mut batch = CommitBatch::empty(); + let mut batch = AsyncCommitBatch::empty(); batch.inbox_receipts.push(InboxReceipt::new("proj", "m1")); - repo.commit_batch(batch).unwrap(); + block_on(repo.commit_batch_async(batch)).unwrap(); assert!(repo.inbox_contains("proj", "m1")); assert!(!repo.inbox_contains("proj", "m2")); // A batch with a duplicate (m1) and a fresh receipt (m2) rolls back whole. - let mut dup = CommitBatch::empty(); + let mut dup = AsyncCommitBatch::empty(); dup.inbox_receipts.push(InboxReceipt::new("proj", "m1")); dup.inbox_receipts.push(InboxReceipt::new("proj", "m2")); - let err = repo.commit_batch(dup).unwrap_err(); + let err = block_on(repo.commit_batch_async(dup)).unwrap_err(); assert!( matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if message_id == "m1"), "got {err:?}" @@ -719,10 +567,10 @@ mod tests { ); // An empty receipt field is rejected (parity with the SQL CHECK). - let mut invalid = CommitBatch::empty(); + let mut invalid = AsyncCommitBatch::empty(); invalid.inbox_receipts.push(InboxReceipt::new("", "m3")); assert!(matches!( - repo.commit_batch(invalid).unwrap_err(), + block_on(repo.commit_batch_async(invalid)).unwrap_err(), RepositoryError::InvalidInboxReceipt { .. } )); } diff --git a/src/lib.rs b/src/lib.rs index 428a0a19a..fc4d30772 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,16 +39,12 @@ pub type SourcedResult = std::result::Result; pub use repository::{ AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, AsyncSnapshotStore, AsyncSnapshotWrite, - AsyncStreamWrite, AsyncTransactionalCommit, Commit, CommitBatch, Get, GetMany, GetOne, - Gettable, InboxOutcome, InboxReceipt, PreparedEventAppend, Repository, RepositoryError, - SnapshotWrite, StreamIdentity, TransactionalCommit, + AsyncStreamWrite, AsyncTransactionalCommit, CommitBatch, InboxOutcome, InboxReceipt, + PreparedEventAppend, RepositoryError, SnapshotWrite, StreamIdentity, }; // Re-export aggregate types at crate root for convenience -pub use aggregate::{ - hydrate, Aggregate, AggregateBuilder, AggregateRepository, AsyncAggregateBuilder, - AsyncAggregateRepository, CommitAggregate, GetAggregate, GetAllAggregates, RepositoryExt, -}; +pub use aggregate::{hydrate, Aggregate, AsyncAggregateBuilder, AsyncAggregateRepository}; pub use hashmap_repo::{HashMapOutboxStore, HashMapRepository}; #[cfg(feature = "postgres")] @@ -59,14 +55,14 @@ pub use sqlite_repo::{SqliteOutboxStore, SqliteRepository}; // Re-export lock traits and types at crate root for convenience pub use lock::{ AsyncLock, AsyncLockManager, InMemoryAsyncLock, InMemoryAsyncLockFuture, - InMemoryAsyncLockManager, InMemoryLock, InMemoryLockManager, Lock, LockError, LockManager, + InMemoryAsyncLockManager, LockError, }; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ outbox_message_insert_plan, outbox_message_key, outbox_message_row_values, - outbox_message_schema, AsyncOutboxCommit, OutboxMessage, OutboxMessageStatus, SyncOutboxCommit, - SyncOutboxCommitExt, OUTBOX_MESSAGES_TABLE, + outbox_message_schema, AsyncOutboxCommit, OutboxMessage, OutboxMessageStatus, + OUTBOX_MESSAGES_TABLE, }; // Outbox Worker: drain and publish concerns @@ -95,14 +91,10 @@ pub use queued_repo::{ AsyncGetAllWithOpts, AsyncGetWithOpts, AsyncUnlockableRepository, - // WithOpts traits for opting out of locking - GetAllWithOpts, - GetWithOpts, // Queued repository Queueable, QueuedRepository, ReadOpts, - UnlockableRepository, }; // Read models: projections and read-optimized views @@ -114,10 +106,9 @@ pub use read_model::{ ReadModelMigrationArtifact, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, ReadModelSchemaAdapter, ReadModelSchemaAdapterCapabilities, ReadModelSchemaBootstrap, ReadModelSchemaIssue, ReadModelSchemaIssueKind, ReadModelSchemaRegistry, - ReadModelSchemaVerification, ReadModelWorkspace, ReadModelWorkspaceExt, ReadModelWritePlan, - ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModel, - RelationalReadModelIncludes, RelationalReadModelQueryStore, RelationshipDef, RelationshipKind, - RowKey, RowMutation, RowPatch, RowValue, RowValues, RowWriteMode, Versioned, + ReadModelSchemaVerification, ReadModelWorkspace, ReadModelWritePlan, ReadModelWritePlanBuilder, + RelationalReadModel, RelationalReadModelIncludes, RelationshipDef, RelationshipKind, RowKey, + RowMutation, RowPatch, RowValue, RowValues, RowWriteMode, Versioned, DEFAULT_READ_MODEL_VERSION_COLUMN, }; @@ -132,17 +123,16 @@ pub use table::{ TableStoreError, TableWritePlan, DEFAULT_TABLE_VERSION_COLUMN, }; -// SyncCommitBuilder: transactional batches of read models, outbox, and aggregates +// AsyncCommitBuilder: transactional batches of read models, outbox, and aggregates pub use commit_builder::{ AsyncCommitBuilder, AsyncCommitBuilderExt, AsyncReadModelWritePlanCommitExt, - AsyncStagedCommitBuilder, SyncCommitBuilder, SyncCommitBuilderExt, - SyncReadModelWritePlanCommitExt, SyncStagedCommitBuilder, + AsyncStagedCommitBuilder, }; // Snapshot: state snapshot payloads and rebuildable cache records for hydration pub use snapshot::{ - hydrate_from_snapshot, AsyncSnapshotAggregateRepository, InMemorySnapshotStore, - SnapshotAggregateRepository, SnapshotRecord, SnapshotStore, Snapshottable, + hydrate_from_snapshot, AsyncSnapshotAggregateRepository, InMemorySnapshotStore, SnapshotRecord, + Snapshottable, }; // Re-export the EventEmitter from the event_emitter_rs crate (requires "emitter" feature) diff --git a/src/lock/in_memory.rs b/src/lock/in_memory.rs deleted file mode 100644 index 530475317..000000000 --- a/src/lock/in_memory.rs +++ /dev/null @@ -1,182 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Condvar, Mutex}; - -use super::{Lock, LockError, LockManager}; - -/// In-memory lock backed by `Mutex` + `Condvar`. -/// -/// This is the default lock implementation — the same logic that was -/// previously the concrete `Lock` struct, now behind the `Lock` trait. -pub struct InMemoryLock { - state: Mutex, - wake: Condvar, -} - -impl InMemoryLock { - pub fn new() -> Self { - InMemoryLock { - state: Mutex::new(false), - wake: Condvar::new(), - } - } -} - -impl Default for InMemoryLock { - fn default() -> Self { - Self::new() - } -} - -impl Lock for InMemoryLock { - fn lock(&self) -> Result<(), LockError> { - let mut locked = self - .state - .lock() - .map_err(|e| LockError::Poisoned(e.to_string()))?; - while *locked { - locked = self - .wake - .wait(locked) - .map_err(|e| LockError::Poisoned(e.to_string()))?; - } - *locked = true; - Ok(()) - } - - fn try_lock(&self) -> Result { - let mut locked = self - .state - .lock() - .map_err(|e| LockError::Poisoned(e.to_string()))?; - if *locked { - Ok(false) - } else { - *locked = true; - Ok(true) - } - } - - fn unlock(&self) -> Result<(), LockError> { - let mut locked = self - .state - .lock() - .map_err(|e| LockError::Poisoned(e.to_string()))?; - if *locked { - *locked = false; - self.wake.notify_one(); - } - Ok(()) - } -} - -/// In-memory lock manager backed by a `HashMap>`. -/// -/// This is the default `LockManager` — it lazily creates one `InMemoryLock` -/// per unique key and returns the same `Arc` for repeated lookups. -pub struct InMemoryLockManager { - locks: Mutex>>, -} - -impl InMemoryLockManager { - pub fn new() -> Self { - InMemoryLockManager { - locks: Mutex::new(HashMap::new()), - } - } -} - -impl Default for InMemoryLockManager { - fn default() -> Self { - Self::new() - } -} - -impl LockManager for InMemoryLockManager { - type Lock = InMemoryLock; - - fn get_lock(&self, id: &str) -> Result, LockError> { - let mut locks = self - .locks - .lock() - .map_err(|_| LockError::Poisoned("lock manager map poisoned".into()))?; - Ok(locks - .entry(id.to_string()) - .or_insert_with(|| Arc::new(InMemoryLock::new())) - .clone()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ======================================================================== - // InMemoryLock tests (migrated from old lock/mod.rs) - // ======================================================================== - - #[test] - fn test_lock_new() { - let lock = InMemoryLock::new(); - assert!(lock.try_lock().unwrap()); // unlocked by default - lock.unlock().unwrap(); - } - - #[test] - fn test_lock_lock() { - let lock = InMemoryLock::new(); - lock.lock().unwrap(); - assert!(!lock.try_lock().unwrap()); // already locked - lock.unlock().unwrap(); - } - - #[test] - fn test_lock_try_lock() { - let lock = InMemoryLock::new(); - assert!(lock.try_lock().unwrap()); - assert!(!lock.try_lock().unwrap()); - lock.unlock().unwrap(); - assert!(lock.try_lock().unwrap()); - lock.unlock().unwrap(); - } - - #[test] - fn test_lock_unlock() { - let lock = InMemoryLock::new(); - lock.lock().unwrap(); - lock.unlock().unwrap(); - assert!(lock.try_lock().unwrap()); // can lock again after unlock - lock.unlock().unwrap(); - } - - // ======================================================================== - // InMemoryLockManager tests - // ======================================================================== - - #[test] - fn same_id_returns_same_arc() { - let manager = InMemoryLockManager::new(); - let lock1 = manager.get_lock("entity-1").unwrap(); - let lock2 = manager.get_lock("entity-1").unwrap(); - assert!(Arc::ptr_eq(&lock1, &lock2)); - } - - #[test] - fn different_id_returns_different_arc() { - let manager = InMemoryLockManager::new(); - let lock1 = manager.get_lock("entity-1").unwrap(); - let lock2 = manager.get_lock("entity-2").unwrap(); - assert!(!Arc::ptr_eq(&lock1, &lock2)); - } - - #[test] - fn manager_locks_are_functional() { - let manager = InMemoryLockManager::new(); - let lock = manager.get_lock("test").unwrap(); - - assert!(lock.try_lock().unwrap()); - assert!(!lock.try_lock().unwrap()); - lock.unlock().unwrap(); - assert!(lock.try_lock().unwrap()); - lock.unlock().unwrap(); - } -} diff --git a/src/lock/lock.rs b/src/lock/lock.rs deleted file mode 100644 index ab43276b4..000000000 --- a/src/lock/lock.rs +++ /dev/null @@ -1,18 +0,0 @@ -use super::LockError; - -/// Trait for a single lock instance. -/// -/// Implementations provide blocking lock, non-blocking try-lock, and unlock. -/// In-memory locks use `Mutex` + `Condvar`; distributed locks might use -/// Redis, Postgres advisory locks, etcd leases, etc. -pub trait Lock: Send + Sync { - /// Acquire the lock, blocking until it becomes available. - fn lock(&self) -> Result<(), LockError>; - - /// Try to acquire the lock without blocking. - /// Returns `Ok(true)` if acquired, `Ok(false)` if already held. - fn try_lock(&self) -> Result; - - /// Release the lock. - fn unlock(&self) -> Result<(), LockError>; -} diff --git a/src/lock/lock_manager.rs b/src/lock/lock_manager.rs deleted file mode 100644 index 252f507fc..000000000 --- a/src/lock/lock_manager.rs +++ /dev/null @@ -1,20 +0,0 @@ -use std::sync::Arc; - -use super::{Lock, LockError}; - -/// Factory trait for obtaining per-entity (or per-key) locks. -/// -/// `QueuedRepository` uses a `LockManager` to -/// obtain a lock for each entity or read model instance. The default -/// `InMemoryLockManager` stores locks in a `HashMap`; distributed -/// implementations might talk to Redis, Postgres, etc. -pub trait LockManager: Send + Sync { - /// The concrete lock type returned by this manager. - type Lock: Lock; - - /// Get (or create) a lock for the given identifier. - /// - /// Repeated calls with the same `id` must return the same logical lock - /// (i.e. the same `Arc` for in-memory, or the same distributed key). - fn get_lock(&self, id: &str) -> Result, LockError>; -} diff --git a/src/lock/mod.rs b/src/lock/mod.rs index 389bb874d..75ef53f29 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -28,14 +28,8 @@ mod async_in_memory; mod async_lock; mod async_lock_manager; mod error; -mod in_memory; -mod lock; -mod lock_manager; pub use async_in_memory::{InMemoryAsyncLock, InMemoryAsyncLockFuture, InMemoryAsyncLockManager}; pub use async_lock::AsyncLock; pub use async_lock_manager::AsyncLockManager; pub use error::LockError; -pub use in_memory::{InMemoryLock, InMemoryLockManager}; -pub use lock::Lock; -pub use lock_manager::LockManager; diff --git a/src/microsvc/transport/outbox_dispatch.rs b/src/microsvc/transport/outbox_dispatch.rs index 05f685d70..1c437ce15 100644 --- a/src/microsvc/transport/outbox_dispatch.rs +++ b/src/microsvc/transport/outbox_dispatch.rs @@ -239,7 +239,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{HashMapRepository, TransactionalCommit}; + use crate::{AsyncCommitBatch, AsyncTransactionalCommit, HashMapRepository}; use std::future::Future; use std::sync::Mutex; @@ -296,9 +296,9 @@ mod tests { fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { let id = message.id().to_string(); - let mut batch = crate::CommitBatch::empty(); + let mut batch = AsyncCommitBatch::empty(); batch.outbox_messages.push(message); - repo.commit_batch(batch).unwrap(); + block_on(repo.commit_batch_async(batch)).unwrap(); id } diff --git a/src/microsvc/transport/outbox_source.rs b/src/microsvc/transport/outbox_source.rs index 7ed487a22..de58ebb63 100644 --- a/src/microsvc/transport/outbox_source.rs +++ b/src/microsvc/transport/outbox_source.rs @@ -178,8 +178,8 @@ mod tests { use crate::microsvc::transport::{run_source, RunOptions}; use crate::microsvc::Service; use crate::{ - CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, OutboxStore, - TransactionalCommit, + AsyncCommitBatch, AsyncTransactionalCommit, HashMapRepository, OutboxMessage, + OutboxMessageStatus, OutboxStore, }; use serde_json::json; use std::future::Future; @@ -205,9 +205,9 @@ mod tests { fn store_row(repo: &HashMapRepository, id: &str, name: &str) { let message = OutboxMessage::create(id, name, b"{}".to_vec()).unwrap(); - let mut batch = CommitBatch::empty(); + let mut batch = AsyncCommitBatch::empty(); batch.outbox_messages.push(message); - repo.commit_batch(batch).unwrap(); + block_on(repo.commit_batch_async(batch)).unwrap(); } fn status(repo: &HashMapRepository, id: &str) -> Option { diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 9434eef29..d00bb58f3 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -1,59 +1,14 @@ -use crate::aggregate::{Aggregate, AggregateRepository, AsyncAggregateRepository}; +use crate::aggregate::{Aggregate, AsyncAggregateRepository}; use crate::outbox::OutboxMessage; use crate::repository::{ - AsyncCommitBatch, AsyncStreamWrite, AsyncTransactionalCommit, CommitBatch, RepositoryError, - StreamIdentity, TransactionalCommit, + AsyncCommitBatch, AsyncStreamWrite, AsyncTransactionalCommit, RepositoryError, StreamIdentity, }; -/// Helper returned by [`SyncOutboxCommitExt::outbox_sync`] to commit an aggregate and outbox -/// message in the same transactional commit batch. -pub struct SyncOutboxCommit<'a, R, A> { - repo: &'a AggregateRepository, - message: OutboxMessage, -} - -impl<'a, R, A> SyncOutboxCommit<'a, R, A> -where - R: TransactionalCommit, - A: Aggregate, -{ - /// Commit the aggregate and outbox message together. - pub fn commit_sync(mut self, aggregate: &mut A) -> Result<(), RepositoryError> { - self.message.set_source(aggregate); - let mut batch = CommitBatch::new(vec![aggregate.entity_mut()]); - batch.outbox_messages.push(self.message); - self.repo.repo().commit_batch(batch) - } -} - -/// Extension trait for aggregate repositories to commit outbox messages alongside aggregates. -pub trait SyncOutboxCommitExt -where - R: TransactionalCommit, - A: Aggregate, -{ - /// Attach an outbox message to be committed with the aggregate. - fn outbox_sync<'a>(&'a self, message: OutboxMessage) -> SyncOutboxCommit<'a, R, A>; -} - -impl SyncOutboxCommitExt for AggregateRepository -where - R: TransactionalCommit, - A: Aggregate, -{ - fn outbox_sync<'a>(&'a self, message: OutboxMessage) -> SyncOutboxCommit<'a, R, A> { - SyncOutboxCommit { - repo: self, - message, - } - } -} - /// Helper returned by [`AsyncAggregateRepository::outbox`] to commit an aggregate /// and an outbox row in the same async transactional batch. /// -/// Borrows the repository (mirroring the synchronous [`outbox_sync`](AsyncOutboxCommitExt)) -/// so it can be called through `ctx.repo()` inside async handlers. +/// Borrows the repository so it can be called through `ctx.repo()` inside async +/// handlers. pub struct AsyncOutboxCommit<'a, R, A> { repo: &'a AsyncAggregateRepository, message: OutboxMessage, @@ -88,11 +43,27 @@ impl AsyncAggregateRepository { #[cfg(test)] mod tests { use super::*; - use crate::{ - sourced, AggregateBuilder, CommitBatch, Entity, HashMapRepository, OutboxStore, - TransactionalCommit, - }; - use std::cell::RefCell; + use crate::{sourced, AsyncAggregateBuilder, Entity, HashMapRepository, OutboxStore}; + use std::sync::Mutex; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } #[derive(Default)] struct Dummy { @@ -111,37 +82,42 @@ mod tests { #[derive(Default)] struct FailingOutboxRepo { - seen_ids: RefCell>, + seen_ids: Mutex>, } - impl TransactionalCommit for FailingOutboxRepo { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - *self.seen_ids.borrow_mut() = batch - .entities - .iter() - .map(|entity| entity.id().to_string()) - .chain( - batch - .outbox_messages - .iter() - .map(|message| message.id().to_string()), - ) - .collect(); - - Err(RepositoryError::Model("outbox write failed".into())) + impl AsyncTransactionalCommit for FailingOutboxRepo { + async fn commit_batch_async<'a>( + &'a self, + batch: AsyncCommitBatch<'a>, + ) -> Result<(), RepositoryError> { + { + *self.seen_ids.lock().unwrap() = batch + .streams + .iter() + .map(|stream| stream.entity.id().to_string()) + .chain( + batch + .outbox_messages + .iter() + .map(|message| message.id().to_string()), + ) + .collect(); + + Err(RepositoryError::Model("outbox write failed".into())) + } } } #[test] fn outbox_helper_commits_both_entities() { - let repo = HashMapRepository::new().aggregate::(); + let repo = HashMapRepository::new().async_aggregate::(); let mut aggregate = Dummy::default(); aggregate.touch().unwrap(); let event = OutboxMessage::create("msg-1", "DummyTouched", b"{}".to_vec()).unwrap(); - repo.outbox_sync(event).commit_sync(&mut aggregate).unwrap(); + block_on(repo.outbox(event).commit(&mut aggregate)).unwrap(); let pending = repo.repo().outbox_store().pending().unwrap(); assert_eq!(pending.len(), 1); @@ -150,23 +126,20 @@ mod tests { #[test] fn outbox_helper_failure_leaves_entities_uncommitted() { - let repo = AggregateRepository::<_, Dummy>::new(FailingOutboxRepo::default()); + let repo = AsyncAggregateRepository::<_, Dummy>::new(FailingOutboxRepo::default()); let mut aggregate = Dummy::default(); aggregate.touch().unwrap(); let event = OutboxMessage::create("msg-fail", "DummyTouched", b"{}".to_vec()).unwrap(); - let err = repo - .outbox_sync(event) - .commit_sync(&mut aggregate) - .unwrap_err(); + let err = block_on(repo.outbox(event).commit(&mut aggregate)).unwrap_err(); assert_eq!(err, RepositoryError::Model("outbox write failed".into())); assert_eq!(aggregate.entity.committed_version(), 0); assert_eq!(aggregate.entity.new_events().len(), 1); assert_eq!( - repo.repo().seen_ids.borrow().as_slice(), + repo.repo().seen_ids.lock().unwrap().as_slice(), &["dummy-1".to_string(), "msg-fail".to_string()] ); } diff --git a/src/outbox/mod.rs b/src/outbox/mod.rs index b3f54de70..1483ce2d5 100644 --- a/src/outbox/mod.rs +++ b/src/outbox/mod.rs @@ -3,8 +3,7 @@ //! This module provides the outbox message type and commit helpers: //! - `OutboxMessage` - publishable message envelope plus delivery state //! - `OutboxMessageStatus` - Message status (Pending, InFlight, Published, Failed) -//! - `SyncOutboxCommit` - Helper for aggregate + outbox commits -//! - `SyncOutboxCommitExt` - Extension trait for repositories +//! - `AsyncOutboxCommit` - Helper for aggregate + outbox commits //! //! Outbox messages are durable publication work items. Their payload can be a //! domain event, integration event, command, or generic transport message. @@ -25,7 +24,7 @@ //! ## Example //! //! ```ignore -//! use sourced_rust::{OutboxMessage, SyncOutboxCommitExt}; +//! use sourced_rust::OutboxMessage; //! //! // Create aggregate and domain event outbox message //! let mut order = Order::new(); @@ -33,8 +32,8 @@ //! //! let outbox = OutboxMessage::create("order-1:created", "OrderCreated", payload); //! -//! // Commit in one repository batch -//! repo.outbox_sync(outbox).commit_sync(&mut order)?; +//! // Commit in one async repository batch +//! repo.outbox(outbox).commit(&mut order).await?; //! ``` mod commit; @@ -50,4 +49,4 @@ pub use table::{ }; // Commit helpers -pub use commit::{AsyncOutboxCommit, SyncOutboxCommit, SyncOutboxCommitExt}; +pub use commit::AsyncOutboxCommit; diff --git a/src/outbox_worker/store.rs b/src/outbox_worker/store.rs index f15cc6ac7..2ca74d899 100644 --- a/src/outbox_worker/store.rs +++ b/src/outbox_worker/store.rs @@ -529,15 +529,35 @@ impl AsyncOutboxStore for HashMapOutboxStore { #[cfg(test)] mod tests { use super::*; - use crate::{HashMapRepository, TransactionalCommit}; + use crate::{AsyncCommitBatch, AsyncTransactionalCommit, HashMapRepository}; + use std::future::Future; use std::sync::{Arc, Barrier}; use std::thread; + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { let id = message.id().to_string(); - let mut batch = crate::CommitBatch::empty(); + let mut batch = AsyncCommitBatch::empty(); batch.outbox_messages.push(message); - repo.commit_batch(batch).unwrap(); + block_on(repo.commit_batch_async(batch)).unwrap(); id } diff --git a/src/queued_repo/mod.rs b/src/queued_repo/mod.rs index 97c334b92..9ca0f2ae8 100644 --- a/src/queued_repo/mod.rs +++ b/src/queued_repo/mod.rs @@ -1,6 +1,6 @@ mod repository; pub use repository::{ - AsyncGetAllWithOpts, AsyncGetWithOpts, AsyncUnlockableRepository, GetAllWithOpts, GetWithOpts, - Queueable, QueuedRepository, ReadOpts, UnlockableRepository, + AsyncGetAllWithOpts, AsyncGetWithOpts, AsyncUnlockableRepository, Queueable, QueuedRepository, + ReadOpts, }; diff --git a/src/queued_repo/repository.rs b/src/queued_repo/repository.rs index 0fff2f617..43807b2b1 100644 --- a/src/queued_repo/repository.rs +++ b/src/queued_repo/repository.rs @@ -6,21 +6,18 @@ use std::future::Future; use std::sync::Arc; -use crate::entity::{Committable, Entity}; -use crate::lock::{ - AsyncLock, AsyncLockManager, InMemoryAsyncLockManager, InMemoryLockManager, Lock, LockError, - LockManager, -}; +use crate::entity::Entity; +use crate::lock::{AsyncLock, AsyncLockManager, InMemoryAsyncLockManager}; use crate::read_model::{ ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, }; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, - AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncTransactionalCommit, Commit, - CommitBatch, Get, GetMany, GetOne, RepositoryError, StreamIdentity, TransactionalCommit, + AsyncRelationalReadModelQueryStore, AsyncSnapshotStore, AsyncTransactionalCommit, + RepositoryError, StreamIdentity, }; -use crate::snapshot::{SnapshotRecord, SnapshotStore}; +use crate::snapshot::SnapshotRecord; /// Options for read operations. #[derive(Debug, Clone, Copy)] @@ -53,7 +50,7 @@ impl ReadOpts { /// Commit releases held locks only after the inner repository succeeds. On /// commit errors, locks remain held so callers can inspect state, retry, or /// explicitly abort. -pub struct QueuedRepository { +pub struct QueuedRepository { inner: R, lock_manager: Arc, } @@ -71,7 +68,7 @@ impl QueuedRepository { pub fn new(inner: R) -> Self { QueuedRepository { inner, - lock_manager: Arc::new(InMemoryLockManager::new()), + lock_manager: Arc::new(InMemoryAsyncLockManager::new()), } } } @@ -88,188 +85,6 @@ impl QueuedRepository { } } -impl QueuedRepository { - /// Create a `QueuedRepository` with a custom lock manager. - pub fn with_lock_manager(inner: R, lock_manager: L) -> Self { - QueuedRepository { - inner, - lock_manager: Arc::new(lock_manager), - } - } - - pub fn lock(&self, id: impl AsRef) -> Result<(), RepositoryError> { - let id = id.as_ref(); - let lock = self.ensure_lock(id)?; - if !lock.try_lock()? { - return Err(LockError::AcquireFailed(format!("lock for {id} is already held")).into()); - } - Ok(()) - } - - pub fn unlock(&self, id: impl AsRef) -> Result<(), RepositoryError> { - let lock = self.ensure_lock(id.as_ref())?; - lock.unlock()?; - Ok(()) - } - - pub fn abort(&self, id: impl AsRef) -> Result<(), RepositoryError> { - self.unlock(id) - } - - fn ensure_lock(&self, id: &str) -> Result, RepositoryError> { - Ok(self.lock_manager.get_lock(id)?) - } - - fn lock_ids_in_order(&self, ids: &[&str]) -> Result>, RepositoryError> { - let mut unique: Vec<&str> = ids.to_vec(); - unique.sort_unstable(); - unique.dedup(); - - let mut locks = Vec::with_capacity(unique.len()); - for id in unique { - let lock = self.ensure_lock(id)?; - lock.lock()?; - locks.push(lock); - } - - Ok(locks) - } -} - -// ============================================================================ -// Core trait implementations (with locking by default) -// ============================================================================ - -impl GetOne for QueuedRepository { - fn get_one(&self, id: &str) -> Result, RepositoryError> { - let lock = self.ensure_lock(id)?; - lock.lock()?; - self.inner.get_one(id) - } -} - -impl GetMany for QueuedRepository { - fn get_many(&self, ids: &[&str]) -> Result, RepositoryError> { - let _locks = self.lock_ids_in_order(ids)?; - self.inner.get_many(ids) - } -} - -impl Commit for QueuedRepository { - fn commit(&self, committable: &mut C) -> Result<(), RepositoryError> { - let entities = committable.entities_mut(); - - // Commit releases locks that were acquired by a prior locking read or - // manual lock call. It does not acquire ownership itself because this - // lock implementation has no guard token or owner tracking. - let mut locks = Vec::with_capacity(entities.len()); - for entity in &entities { - locks.push(self.ensure_lock(entity.id())?); - } - - // Delegate to inner repository - let result = self.inner.commit(committable); - - // Keep locks held on errors so callers can retry or explicitly abort. - if result.is_ok() { - for lock in locks { - lock.unlock()?; - } - } - - result - } -} - -impl TransactionalCommit for QueuedRepository { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - let ids: Vec<&str> = batch.entities.iter().map(|entity| entity.id()).collect(); - // See `Commit::commit`: these handles are released after successful - // inner commit and intentionally kept held on errors. - let mut locks = Vec::with_capacity(ids.len()); - for id in ids { - locks.push(self.ensure_lock(id)?); - } - - let result = self.inner.commit_batch(batch); - - if result.is_ok() { - for lock in locks { - lock.unlock()?; - } - } - - result - } -} - -// ============================================================================ -// WithOpts traits for opting out of locking -// ============================================================================ - -/// Get a single entity with options. -pub trait GetWithOpts: Get { - fn get_with(&self, id: &str, opts: ReadOpts) -> Result, RepositoryError>; -} - -/// Get multiple entities with options. -pub trait GetAllWithOpts: Get { - fn get_all_with(&self, ids: &[&str], opts: ReadOpts) -> Result, RepositoryError>; -} - -impl GetWithOpts for QueuedRepository { - fn get_with(&self, id: &str, opts: ReadOpts) -> Result, RepositoryError> { - if opts.lock { - self.get_one(id) - } else { - self.inner.get_one(id) - } - } -} - -impl GetAllWithOpts for QueuedRepository { - fn get_all_with(&self, ids: &[&str], opts: ReadOpts) -> Result, RepositoryError> { - if opts.lock { - self.get_many(ids) - } else { - self.inner.get_many(ids) - } - } -} - -// ============================================================================ -// Unlock capability -// ============================================================================ - -/// Trait for repositories that support unlocking entities. -pub trait UnlockableRepository { - fn unlock(&self, id: &str) -> Result<(), RepositoryError>; -} - -impl UnlockableRepository for QueuedRepository { - fn unlock(&self, id: &str) -> Result<(), RepositoryError> { - QueuedRepository::unlock(self, id) - } -} - -// ============================================================================ -// SnapshotStore delegation -// ============================================================================ - -impl SnapshotStore for QueuedRepository { - fn get_snapshot(&self, id: &str) -> Result, RepositoryError> { - self.inner.get_snapshot(id) - } - - fn save_snapshot(&self, record: SnapshotRecord) -> Result<(), RepositoryError> { - self.inner.save_snapshot(record) - } - - fn delete_snapshot(&self, id: &str) -> Result { - self.inner.delete_snapshot(id) - } -} - // ============================================================================ // Async variant (async lock manager): the same serialization semantics over // the async repository trait surface. `QueuedRepository` @@ -544,17 +359,9 @@ impl AsyncUnlockableRepository for QueuedRepository QueuedRepository { - QueuedRepository::new(self) - } - - fn queued_with(self, lock_manager: L) -> QueuedRepository { - QueuedRepository::with_lock_manager(self, lock_manager) - } - - /// Wrap with the default async lock manager (the async counterpart to - /// [`queued`](Queueable::queued)). Pair with `.async_aggregate::()` for - /// per-aggregate serialization over the async repository surface. + /// Wrap with the default async lock manager. Pair with + /// `.async_aggregate::()` for per-aggregate serialization over the async + /// repository surface. fn queued_async(self) -> QueuedRepository { QueuedRepository::with_async_lock_manager(self, InMemoryAsyncLockManager::new()) } diff --git a/src/read_model/in_memory.rs b/src/read_model/in_memory.rs index 53c9447a4..1d0592450 100644 --- a/src/read_model/in_memory.rs +++ b/src/read_model/in_memory.rs @@ -13,9 +13,8 @@ use super::{ ExpectedVersion, PatchMode, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, ReadModelSchemaRegistry, - ReadModelWritePlan, ReadModelWritePlanStore, RelationalReadModel, - RelationalReadModelQueryStore, RelationshipDef, RelationshipKind, RowKey, RowValue, RowValues, - RowWriteMode, Versioned, + ReadModelWritePlan, RelationalReadModel, RelationshipDef, RelationshipKind, RowKey, RowValue, + RowValues, RowWriteMode, Versioned, }; use crate::repository::{AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore}; @@ -278,41 +277,30 @@ impl InMemoryReadModelStore { } } -impl ReadModelWritePlanStore for InMemoryReadModelStore { - fn read_model_capabilities(&self) -> ReadModelAdapterCapabilities { - relational_capabilities() - } - - fn commit_write_plan( - &self, - plan: ReadModelWritePlan, - ) -> Result { - let mut relational_rows = self - .relational_rows - .write() - .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; - - let mut staged_rows = relational_rows.clone(); - let outcome = apply_read_model_write_plan(plan, &mut staged_rows)?; - - if outcome.was_applied() { - *relational_rows = staged_rows; - } - - Ok(outcome) - } -} - impl AsyncReadModelWritePlanStore for InMemoryReadModelStore { fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { - ReadModelWritePlanStore::read_model_capabilities(self) + relational_capabilities() } fn commit_write_plan_async( &self, plan: ReadModelWritePlan, ) -> impl Future> + Send + '_ { - async move { ReadModelWritePlanStore::commit_write_plan(self, plan) } + async move { + let mut relational_rows = self + .relational_rows + .write() + .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; + + let mut staged_rows = relational_rows.clone(); + let outcome = apply_read_model_write_plan(plan, &mut staged_rows)?; + + if outcome.was_applied() { + *relational_rows = staged_rows; + } + + Ok(outcome) + } } } @@ -323,69 +311,58 @@ struct IncludeSpec { target_schema: ReadModelSchema, } -impl RelationalReadModelQueryStore for InMemoryReadModelStore { - fn read_model_query_capabilities(&self) -> ReadModelQueryCapabilities { - ReadModelQueryCapabilities::relationship_includes() - } - - fn load_graph( - &self, - request: ReadModelLoadRequest, - ) -> Result { - request.validate_for_query_capabilities(&self.read_model_query_capabilities())?; - - let (root_schema, include_specs) = { - let registry = self - .schema_registry - .read() - .map_err(|_| ReadModelError::Storage("schema registry lock poisoned".into()))?; - resolve_request_schemas(®istry, &request)? - }; - validate_key(&root_schema, &request.key)?; - - let rows = self - .relational_rows - .read() - .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; - let root_storage_key = relational_storage_key(&root_schema.table_name, &request.key); - let Some(root_row) = rows.get(&root_storage_key) else { - return Ok(ReadModelLoadGraph::default()); - }; - let root = Versioned { - data: root_row.values.clone(), - version: root_row.version, - }; - - let mut includes = BTreeMap::new(); - for spec in include_specs { - let loaded_rows = load_relationship_rows(&rows, &root_schema, &root.data, &spec)?; - includes.insert( - spec.name, - ReadModelIncludeRows { - relationship: spec.relationship, - target_schema: spec.target_schema, - rows: loaded_rows, - }, - ); - } - - Ok(ReadModelLoadGraph { - root: Some(root), - includes, - }) - } -} - impl AsyncRelationalReadModelQueryStore for InMemoryReadModelStore { fn read_model_query_capabilities_async(&self) -> ReadModelQueryCapabilities { - RelationalReadModelQueryStore::read_model_query_capabilities(self) + ReadModelQueryCapabilities::relationship_includes() } fn load_graph_async( &self, request: ReadModelLoadRequest, ) -> impl Future> + Send + '_ { - async move { RelationalReadModelQueryStore::load_graph(self, request) } + async move { + request.validate_for_query_capabilities(&self.read_model_query_capabilities_async())?; + + let (root_schema, include_specs) = { + let registry = self + .schema_registry + .read() + .map_err(|_| ReadModelError::Storage("schema registry lock poisoned".into()))?; + resolve_request_schemas(®istry, &request)? + }; + validate_key(&root_schema, &request.key)?; + + let rows = self + .relational_rows + .read() + .map_err(|_| ReadModelError::Storage("lock poisoned".into()))?; + let root_storage_key = relational_storage_key(&root_schema.table_name, &request.key); + let Some(root_row) = rows.get(&root_storage_key) else { + return Ok(ReadModelLoadGraph::default()); + }; + let root = Versioned { + data: root_row.values.clone(), + version: root_row.version, + }; + + let mut includes = BTreeMap::new(); + for spec in include_specs { + let loaded_rows = load_relationship_rows(&rows, &root_schema, &root.data, &spec)?; + includes.insert( + spec.name, + ReadModelIncludeRows { + relationship: spec.relationship, + target_schema: spec.target_schema, + rows: loaded_rows, + }, + ); + } + + Ok(ReadModelLoadGraph { + root: Some(root), + includes, + }) + } } } @@ -593,6 +570,25 @@ mod tests { RowPatch, }; + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + fn test_row_schema() -> ReadModelSchema { ReadModelSchema { model_name: "TestRow".into(), @@ -614,17 +610,16 @@ mod tests { let mut values = RowValues::new(); values.insert("id", RowValue::String("row-1".into())); - let outcome = store - .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( - RowMutation { - schema: schema.clone(), - key: key.clone(), - values, - expected_version: ExpectedVersion::Any, - mode: RowWriteMode::Upsert, - }, - )])) - .unwrap(); + let outcome = block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ + ReadModelMutation::UpsertRow(RowMutation { + schema: schema.clone(), + key: key.clone(), + values, + expected_version: ExpectedVersion::Any, + mode: RowWriteMode::Upsert, + }), + ]))) + .unwrap(); let row = store .relational_rows .read() @@ -649,28 +644,26 @@ mod tests { let mut values = RowValues::new(); values.insert("id", RowValue::String("row-1".into())); - store - .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( - RowMutation { - schema: schema.clone(), - key: key.clone(), - values, - expected_version: ExpectedVersion::Any, - mode: RowWriteMode::Upsert, - }, - )])) - .unwrap(); - store - .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::PatchRow( - PatchRowMutation { - schema: schema.clone(), - key: key.clone(), - patch: RowPatch::new().set("id", RowValue::String("row-1".into())), - expected_version: ExpectedVersion::Exact(1), - mode: PatchMode::UpdateExisting, - }, - )])) - .unwrap(); + block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ + ReadModelMutation::UpsertRow(RowMutation { + schema: schema.clone(), + key: key.clone(), + values, + expected_version: ExpectedVersion::Any, + mode: RowWriteMode::Upsert, + }), + ]))) + .unwrap(); + block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ + ReadModelMutation::PatchRow(PatchRowMutation { + schema: schema.clone(), + key: key.clone(), + patch: RowPatch::new().set("id", RowValue::String("row-1".into())), + expected_version: ExpectedVersion::Exact(1), + mode: PatchMode::UpdateExisting, + }), + ]))) + .unwrap(); let version = store .relational_rows .read() @@ -680,15 +673,14 @@ mod tests { .version; assert_eq!(version, 2); - store - .commit_write_plan(ReadModelWritePlan::new(vec![ReadModelMutation::DeleteRow( - DeleteRowMutation { - schema: schema.clone(), - key: key.clone(), - expected_version: ExpectedVersion::Exact(2), - }, - )])) - .unwrap(); + block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ + ReadModelMutation::DeleteRow(DeleteRowMutation { + schema: schema.clone(), + key: key.clone(), + expected_version: ExpectedVersion::Exact(2), + }), + ]))) + .unwrap(); assert!(!store .relational_rows .read() diff --git a/src/read_model/mod.rs b/src/read_model/mod.rs index a89c0cf96..7ae7089e7 100644 --- a/src/read_model/mod.rs +++ b/src/read_model/mod.rs @@ -3,15 +3,15 @@ //! Relational models stage explicit row mutations: //! //! ```ignore -//! use sourced_rust::{ReadModelWritePlanBuilder, SyncReadModelWritePlanCommitExt}; +//! use sourced_rust::{ReadModelWritePlanBuilder, AsyncReadModelWritePlanCommitExt}; //! //! let mut read_models = ReadModelWritePlanBuilder::new(); //! read_models.upsert(&player)?; //! read_models.upsert_related(&player, "weapons", &weapon)?; -//! repo.read_models_sync(read_models).commit_sync(&mut aggregate)?; +//! repo.read_models(read_models).commit(&mut aggregate).await?; //! ``` //! -//! Async persistent repositories expose the same staging shape through +//! Async persistent repositories expose the staging shape through //! `AsyncReadModelWritePlanCommitExt::read_models`, returning a future //! from `commit`. //! @@ -120,7 +120,6 @@ pub use session::{ AsyncReadModelLoadBuilder, AsyncReadModelWorkspaceExt, DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelMutation, - ReadModelQueryCapabilities, ReadModelWorkspace, ReadModelWorkspaceExt, ReadModelWritePlan, - ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModelQueryStore, RowMutation, - RowPatch, RowWriteMode, + ReadModelQueryCapabilities, ReadModelWorkspace, ReadModelWritePlan, ReadModelWritePlanBuilder, + RowMutation, RowPatch, RowWriteMode, }; diff --git a/src/read_model/session.rs b/src/read_model/session.rs index d4d0ed9ca..d12acf096 100644 --- a/src/read_model/session.rs +++ b/src/read_model/session.rs @@ -83,16 +83,6 @@ impl ReadModelCommitOutcome { } } -/// Adapter contract for committing read-model write plans without an aggregate repository. -pub trait ReadModelWritePlanStore: Send + Sync { - fn read_model_capabilities(&self) -> ReadModelAdapterCapabilities; - - fn commit_write_plan( - &self, - plan: ReadModelWritePlan, - ) -> Result; -} - /// A request an adapter can satisfy with a primary-key read plus explicit includes. #[derive(Clone, Debug, PartialEq)] pub struct ReadModelLoadRequest { @@ -145,16 +135,6 @@ pub struct ReadModelLoadGraph { pub includes: BTreeMap, } -/// Adapter contract for explicit primary-key read-model loads and includes. -pub trait RelationalReadModelQueryStore: Send + Sync { - fn read_model_query_capabilities(&self) -> ReadModelQueryCapabilities; - - fn load_graph( - &self, - request: ReadModelLoadRequest, - ) -> Result; -} - /// Sparse column updates for a relational row. #[derive(Clone, Debug, Default, PartialEq)] pub struct RowPatch { @@ -591,13 +571,6 @@ impl ReadModelWritePlanBuilder { Ok(plan) } - pub fn commit(self, store: &S) -> Result - where - S: ReadModelWritePlanStore + ?Sized, - { - store.commit_write_plan(self.into_write_plan()?) - } - pub async fn commit_async(self, store: &S) -> Result where S: AsyncReadModelWritePlanStore + ?Sized, @@ -754,9 +727,8 @@ const INITIAL_TRACKED_ROW_VERSION: u64 = 1; /// Store-bound read-model workspace for load, mutate, sync, commit workflows. /// -/// The mutation/sync/diff surface is store-independent; `load`/`commit` are -/// provided by sync- and async-store impl blocks below, so the same workspace -/// drives both the sync (`commit`) and async (`commit_async`) store traits. +/// The mutation/sync/diff surface is store-independent; `load_async`/`commit_async` +/// are provided by the async-store impl block below. pub struct ReadModelWorkspace<'a, S> { store: &'a S, writes: ReadModelWritePlanBuilder, @@ -1136,29 +1108,6 @@ impl<'a, S> ReadModelWorkspace<'a, S> { } } -impl<'a, S> ReadModelWorkspace<'a, S> -where - S: ReadModelWritePlanStore + RelationalReadModelQueryStore, -{ - /// Begin a tracked load against the synchronous store traits. - pub fn load(&mut self, key: RowKey) -> ReadModelLoadBuilder<'_, 'a, S, M> - where - M: RelationalReadModel + RelationalReadModelIncludes, - { - ReadModelLoadBuilder { - unit: self, - key, - includes: Vec::new(), - _marker: PhantomData, - } - } - - /// Commit the staged write plan through the synchronous store. - pub fn commit(self) -> Result { - self.writes.commit(self.store) - } -} - impl<'a, S> ReadModelWorkspace<'a, S> where S: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, @@ -1182,67 +1131,6 @@ where } } -/// Builder for one explicit primary-key read-model load. -pub struct ReadModelLoadBuilder<'workspace, 'store, S, M> -where - S: ReadModelWritePlanStore + RelationalReadModelQueryStore, -{ - unit: &'workspace mut ReadModelWorkspace<'store, S>, - key: RowKey, - includes: Vec, - _marker: PhantomData, -} - -impl<'workspace, 'store, S, M> ReadModelLoadBuilder<'workspace, 'store, S, M> -where - S: ReadModelWritePlanStore + RelationalReadModelQueryStore, - M: RelationalReadModel + RelationalReadModelIncludes, -{ - pub fn include(mut self, relationship: impl Into) -> Self { - self.includes.push(relationship.into()); - self - } - - pub fn one(self) -> Result>, ReadModelError> { - let request = self - .unit - .writes - .load_with::(self.key, self.includes)?; - let graph = self.unit.store.load_graph(request.clone())?; - let Some(root) = graph.root else { - return Ok(None); - }; - - let mut model = M::from_row(root.data.clone())?; - for (include_name, include_rows) in &graph.includes { - let rows = include_rows - .rows - .iter() - .map(|row| row.data.clone()) - .collect::>(); - model.hydrate_include(include_name, rows)?; - } - - self.unit - .track_graph(request.schema, root.clone(), graph.includes)?; - Ok(Some(Versioned { - data: model, - version: root.version, - })) - } -} - -/// Extension trait that starts a tracked read-model workspace from a store. -pub trait ReadModelWorkspaceExt: - ReadModelWritePlanStore + RelationalReadModelQueryStore + Sized -{ - fn workspace(&self) -> ReadModelWorkspace<'_, Self> { - ReadModelWorkspace::new(self) - } -} - -impl ReadModelWorkspaceExt for S where S: ReadModelWritePlanStore + RelationalReadModelQueryStore {} - /// Builder for one explicit primary-key read-model load over the async store traits. pub struct AsyncReadModelLoadBuilder<'workspace, 'store, S, M> where diff --git a/src/repository/batch.rs b/src/repository/batch.rs index 0d38386f2..4cf6f9315 100644 --- a/src/repository/batch.rs +++ b/src/repository/batch.rs @@ -4,7 +4,6 @@ use crate::read_model::ReadModelWritePlan; use crate::snapshot::SnapshotRecord; use super::inbox::InboxReceipt; -use super::RepositoryError; /// A snapshot write staged as part of a transactional commit. #[derive(Clone, Debug)] @@ -38,8 +37,3 @@ impl<'a> CommitBatch<'a> { Self::new(Vec::new()) } } - -/// Repository capability for writes that must commit or roll back together. -pub trait TransactionalCommit { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError>; -} diff --git a/src/repository/gettable.rs b/src/repository/gettable.rs deleted file mode 100644 index 767f65565..000000000 --- a/src/repository/gettable.rs +++ /dev/null @@ -1,150 +0,0 @@ -use super::RepositoryError; -use crate::entity::Entity; - -/// Trait for types that can be used as get arguments. -pub trait Gettable { - type Output; - fn get_from(&self, repo: &R) -> Result; -} - -/// Internal trait for getting a single entity. -pub trait GetOne { - fn get_one(&self, id: &str) -> Result, RepositoryError>; -} - -/// Internal trait for getting multiple entities. -pub trait GetMany { - fn get_many(&self, ids: &[&str]) -> Result, RepositoryError>; -} - -// Single ID (&str) -impl Gettable for &str { - type Output = Option; - - fn get_from(&self, repo: &R) -> Result { - repo.get_one(self) - } -} - -// Single ID (String) -impl Gettable for String { - type Output = Option; - - fn get_from(&self, repo: &R) -> Result { - repo.get_one(self.as_str()) - } -} - -// Single ID (&String) -impl Gettable for &String { - type Output = Option; - - fn get_from(&self, repo: &R) -> Result { - repo.get_one(self.as_str()) - } -} - -// Slice of &str -impl Gettable for &[&str] { - type Output = Vec; - - fn get_from(&self, repo: &R) -> Result { - repo.get_many(self) - } -} - -// Fixed-size arrays -impl Gettable for [&str; N] { - type Output = Vec; - - fn get_from(&self, repo: &R) -> Result { - repo.get_many(self.as_slice()) - } -} - -// Fixed-size arrays by reference -impl Gettable for &[&str; N] { - type Output = Vec; - - fn get_from(&self, repo: &R) -> Result { - repo.get_many(self.as_slice()) - } -} - -// Vec of &str -impl Gettable for Vec<&str> { - type Output = Vec; - - fn get_from(&self, repo: &R) -> Result { - repo.get_many(self.as_slice()) - } -} - -// Vec of &str by reference -impl Gettable for &Vec<&str> { - type Output = Vec; - - fn get_from(&self, repo: &R) -> Result { - repo.get_many(self.as_slice()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - struct MockRepo; - - impl GetOne for MockRepo { - fn get_one(&self, id: &str) -> Result, RepositoryError> { - Ok(Some(Entity::with_id(id))) - } - } - - impl GetMany for MockRepo { - fn get_many(&self, ids: &[&str]) -> Result, RepositoryError> { - Ok(ids.iter().map(|id| Entity::with_id(*id)).collect()) - } - } - - #[test] - fn single_str() { - let repo = MockRepo; - let result: Option = "test-id".get_from(&repo).unwrap(); - assert!(result.is_some()); - assert_eq!(result.unwrap().id(), "test-id"); - } - - #[test] - fn single_string() { - let repo = MockRepo; - let id = String::from("test-id"); - let result: Option = id.get_from(&repo).unwrap(); - assert!(result.is_some()); - assert_eq!(result.unwrap().id(), "test-id"); - } - - #[test] - fn slice_of_ids() { - let repo = MockRepo; - let ids: &[&str] = &["a", "b", "c"]; - let result: Vec = ids.get_from(&repo).unwrap(); - assert_eq!(result.len(), 3); - } - - #[test] - fn array_of_ids() { - let repo = MockRepo; - let ids = ["a", "b"]; - let result: Vec = ids.get_from(&repo).unwrap(); - assert_eq!(result.len(), 2); - } - - #[test] - fn vec_of_ids() { - let repo = MockRepo; - let ids = vec!["a", "b", "c", "d"]; - let result: Vec = ids.get_from(&repo).unwrap(); - assert_eq!(result.len(), 4); - } -} diff --git a/src/repository/mod.rs b/src/repository/mod.rs index a055b2ca9..141ea0084 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -1,19 +1,15 @@ mod async_repository; mod batch; mod error; -mod gettable; mod identity; mod inbox; -mod repository; pub use async_repository::{ AsyncCommitBatch, AsyncGetStream, AsyncInboxStore, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, AsyncRepository, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, AsyncTransactionalCommit, PreparedEventAppend, }; -pub use batch::{CommitBatch, SnapshotWrite, TransactionalCommit}; +pub use batch::{CommitBatch, SnapshotWrite}; pub use error::RepositoryError; -pub use gettable::{GetMany, GetOne, Gettable}; pub use identity::StreamIdentity; pub use inbox::{InboxOutcome, InboxReceipt}; -pub use repository::{Commit, Get, Repository}; diff --git a/src/repository/repository.rs b/src/repository/repository.rs deleted file mode 100644 index 08bc2ccc1..000000000 --- a/src/repository/repository.rs +++ /dev/null @@ -1,28 +0,0 @@ -use super::error::RepositoryError; -use super::gettable::{GetMany, GetOne, Gettable}; - -/// Get one or more aggregate event streams by ID. -pub trait Get: GetOne + GetMany { - fn get(&self, gettable: G) -> Result - where - Self: Sized, - { - gettable.get_from(self) - } -} - -// Blanket implementation: anything implementing GetOne + GetMany is Get -impl Get for T {} - -use crate::entity::Committable; - -/// Append new aggregate event records for one or more entities. -pub trait Commit { - fn commit(&self, committable: &mut C) -> Result<(), RepositoryError>; -} - -/// Repository trait for types that implement both read-by-ID and commit APIs. -pub trait Repository: Get + Commit {} - -// Blanket implementation: anything implementing Get and Commit is a Repository. -impl Repository for T where T: Get + Commit {} diff --git a/src/snapshot/in_memory.rs b/src/snapshot/in_memory.rs index d4cfd8e00..1febd38e1 100644 --- a/src/snapshot/in_memory.rs +++ b/src/snapshot/in_memory.rs @@ -9,7 +9,7 @@ use std::sync::{Arc, RwLock}; use crate::repository::{AsyncSnapshotStore, RepositoryError, StreamIdentity}; -use super::store::{SnapshotRecord, SnapshotStore}; +use super::store::SnapshotRecord; /// In-memory snapshot store backed by `Arc>`. /// @@ -34,34 +34,6 @@ impl InMemorySnapshotStore { } } -impl SnapshotStore for InMemorySnapshotStore { - fn get_snapshot(&self, id: &str) -> Result, RepositoryError> { - let storage = self - .storage - .read() - .map_err(|_| RepositoryError::LockPoisoned("snapshot read"))?; - Ok(storage.get(id).cloned()) - } - - fn save_snapshot(&self, record: SnapshotRecord) -> Result<(), RepositoryError> { - record.validate()?; - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("snapshot write"))?; - storage.insert(record.aggregate_id.clone(), record); - Ok(()) - } - - fn delete_snapshot(&self, id: &str) -> Result { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("snapshot write"))?; - Ok(storage.remove(id).is_some()) - } -} - impl AsyncSnapshotStore for InMemorySnapshotStore { fn get_snapshot_async<'a>( &'a self, @@ -110,6 +82,29 @@ impl AsyncSnapshotStore for InMemorySnapshotStore { mod tests { use super::*; + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + fn identity(id: &str) -> StreamIdentity { + StreamIdentity::new("test.aggregate", id).unwrap() + } + #[test] fn save_and_get() { let store = InMemorySnapshotStore::new(); @@ -121,9 +116,11 @@ mod tests { 1, vec![1, 2, 3], ); - store.save_snapshot(record).unwrap(); + block_on(store.save_snapshot_async(&identity("agg-1"), record)).unwrap(); - let loaded = store.get_snapshot("agg-1").unwrap().unwrap(); + let loaded = block_on(store.get_snapshot_async(&identity("agg-1"))) + .unwrap() + .unwrap(); assert_eq!(loaded.version, 5); assert_eq!(loaded.payload, vec![1, 2, 3]); assert_eq!(loaded.snapshot_type, "TestSnapshot"); @@ -132,34 +129,28 @@ mod tests { #[test] fn get_missing_returns_none() { let store = InMemorySnapshotStore::new(); - assert!(store.get_snapshot("missing").unwrap().is_none()); + assert!(block_on(store.get_snapshot_async(&identity("missing"))) + .unwrap() + .is_none()); } #[test] fn save_overwrites() { let store = InMemorySnapshotStore::new(); - store - .save_snapshot(SnapshotRecord::new( - "test.aggregate", - "agg-1", - 1, - "TestSnapshot", - 1, - vec![1], - )) + block_on(store.save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), + )) + .unwrap(); + block_on(store.save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 5, "TestSnapshot", 1, vec![5]), + )) + .unwrap(); + + let loaded = block_on(store.get_snapshot_async(&identity("agg-1"))) + .unwrap() .unwrap(); - store - .save_snapshot(SnapshotRecord::new( - "test.aggregate", - "agg-1", - 5, - "TestSnapshot", - 1, - vec![5], - )) - .unwrap(); - - let loaded = store.get_snapshot("agg-1").unwrap().unwrap(); assert_eq!(loaded.version, 5); assert_eq!(loaded.payload, vec![5]); } @@ -167,42 +158,36 @@ mod tests { #[test] fn delete_existing() { let store = InMemorySnapshotStore::new(); - store - .save_snapshot(SnapshotRecord::new( - "test.aggregate", - "agg-1", - 1, - "TestSnapshot", - 1, - vec![1], - )) - .unwrap(); - assert!(store.delete_snapshot("agg-1").unwrap()); - assert!(store.get_snapshot("agg-1").unwrap().is_none()); + block_on(store.save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), + )) + .unwrap(); + assert!(block_on(store.delete_snapshot_async(&identity("agg-1"))).unwrap()); + assert!(block_on(store.get_snapshot_async(&identity("agg-1"))) + .unwrap() + .is_none()); } #[test] fn delete_missing_returns_false() { let store = InMemorySnapshotStore::new(); - assert!(!store.delete_snapshot("missing").unwrap()); + assert!(!block_on(store.delete_snapshot_async(&identity("missing"))).unwrap()); } #[test] fn clone_shares_storage() { let store = InMemorySnapshotStore::new(); let clone = store.clone(); - store - .save_snapshot(SnapshotRecord::new( - "test.aggregate", - "agg-1", - 3, - "TestSnapshot", - 1, - vec![3], - )) + block_on(store.save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 3, "TestSnapshot", 1, vec![3]), + )) + .unwrap(); + + let loaded = block_on(clone.get_snapshot_async(&identity("agg-1"))) + .unwrap() .unwrap(); - - let loaded = clone.get_snapshot("agg-1").unwrap().unwrap(); assert_eq!(loaded.version, 3); } } diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index 5da3ebaa0..e7b7d6ed8 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -4,8 +4,6 @@ mod snapshottable; mod store; pub use in_memory::InMemorySnapshotStore; -pub use repository::{ - hydrate_from_snapshot, AsyncSnapshotAggregateRepository, SnapshotAggregateRepository, -}; +pub use repository::{hydrate_from_snapshot, AsyncSnapshotAggregateRepository}; pub use snapshottable::Snapshottable; -pub use store::{SnapshotRecord, SnapshotStore}; +pub use store::SnapshotRecord; diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index 82786a827..d42e750b9 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -1,14 +1,12 @@ -use crate::aggregate::{hydrate, AggregateRepository, AsyncAggregateRepository}; +use crate::aggregate::{hydrate, AsyncAggregateRepository}; use crate::entity::{upcast_events, Entity}; -use crate::queued_repo::{GetAllWithOpts, GetWithOpts, ReadOpts, UnlockableRepository}; use crate::repository::{ AsyncCommitBatch, AsyncGetStream, AsyncSnapshotStore, AsyncSnapshotWrite, AsyncStreamWrite, - AsyncTransactionalCommit, CommitBatch, Get, RepositoryError, SnapshotWrite, StreamIdentity, - TransactionalCommit, + AsyncTransactionalCommit, RepositoryError, StreamIdentity, }; use super::snapshottable::Snapshottable; -use super::store::{SnapshotRecord, SnapshotStore}; +use super::store::SnapshotRecord; #[derive(Debug, PartialEq, Eq)] enum SnapshotHydrationError { @@ -171,23 +169,6 @@ fn hydrate_with_optional_snapshot( .map_err(snapshot_hydration_error_to_repository_error) } -/// A repository wrapper that provides snapshot-aware get and commit for a specific aggregate type. -pub struct SnapshotAggregateRepository { - inner: AggregateRepository, - frequency: u64, -} - -impl SnapshotAggregateRepository { - pub fn new(inner: AggregateRepository, frequency: u64) -> Self { - SnapshotAggregateRepository { inner, frequency } - } - - /// Access the inner AggregateRepository. - pub fn repo(&self) -> &AggregateRepository { - &self.inner - } -} - /// Async repository wrapper that treats aggregate snapshots as rebuildable /// hydration cache records. pub struct AsyncSnapshotAggregateRepository { @@ -331,224 +312,10 @@ where } } -// ============================================================================ -// get / get_all — snapshot-aware hydration -// ============================================================================ - -impl SnapshotAggregateRepository -where - R: Get + SnapshotStore, - A: Snapshottable, -{ - /// Load an aggregate, using a snapshot if available. - pub fn get(&self, id: &str) -> Result, RepositoryError> { - let entity = self.inner.repo().get(id)?; - let Some(entity) = entity else { - return Ok(None); - }; - let snapshot = self.inner.repo().get_snapshot(id)?; - Ok(Some(self.hydrate_with_optional_snapshot(entity, snapshot)?)) - } - - /// Load multiple aggregates by ID. - pub fn get_all(&self, ids: &[&str]) -> Result, RepositoryError> { - let entities = self.inner.repo().get(ids)?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - let snapshot = self.inner.repo().get_snapshot(entity.id())?; - aggregates.push(self.hydrate_with_optional_snapshot(entity, snapshot)?); - } - Ok(aggregates) - } - - fn hydrate_with_optional_snapshot( - &self, - entity: Entity, - snapshot: Option, - ) -> Result { - hydrate_with_optional_snapshot::(entity, snapshot) - } -} - -// ============================================================================ -// commit / commit_all — auto-snapshot after threshold -// ============================================================================ - -impl SnapshotAggregateRepository -where - R: TransactionalCommit, - A: Snapshottable, -{ - /// Commit the aggregate and create a snapshot if the frequency threshold is met. - pub fn commit(&self, aggregate: &mut A) -> Result<(), RepositoryError> { - let snapshot = self.snapshot_record(aggregate)?; - let snapshot_version = snapshot.as_ref().map(|record| record.version); - let snapshots = snapshot.into_iter().map(SnapshotWrite::Save).collect(); - - self.inner.repo().commit_batch(CommitBatch { - entities: vec![aggregate.entity_mut()], - outbox_messages: Vec::new(), - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - })?; - - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - Ok(()) - } - - /// Commit multiple aggregates and create snapshots where thresholds are met. - pub fn commit_all(&self, aggregates: &mut [&mut A]) -> Result<(), RepositoryError> { - let mut snapshot_versions = Vec::with_capacity(aggregates.len()); - let mut snapshots = Vec::new(); - for aggregate in aggregates.iter() { - let snapshot = self.snapshot_record(*aggregate)?; - snapshot_versions.push(snapshot.as_ref().map(|record| record.version)); - if let Some(record) = snapshot { - snapshots.push(SnapshotWrite::Save(record)); - } - } - - let entities: Vec<&mut Entity> = aggregates - .iter_mut() - .map(|agg| (*agg).entity_mut()) - .collect(); - self.inner.repo().commit_batch(CommitBatch { - entities, - outbox_messages: Vec::new(), - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - })?; - - for (aggregate, snapshot_version) in aggregates.iter_mut().zip(snapshot_versions) { - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - } - Ok(()) - } - - fn snapshot_record(&self, aggregate: &A) -> Result, RepositoryError> { - let version = aggregate.entity().version(); - let snap_version = aggregate.entity().snapshot_version(); - - if snapshot_due(version, snap_version, self.frequency) { - return snapshot_record_for(aggregate).map(Some); - } - Ok(None) - } -} - -// ============================================================================ -// abort / peek — delegate through inner AggregateRepository -// ============================================================================ - -impl SnapshotAggregateRepository -where - R: UnlockableRepository, - A: Snapshottable, -{ - pub fn abort(&self, aggregate: &A) -> Result<(), RepositoryError> { - self.inner.repo().unlock(aggregate.entity().id()) - } -} - -impl SnapshotAggregateRepository -where - R: GetWithOpts + SnapshotStore, - A: Snapshottable, -{ - /// Non-locking read with snapshot-aware hydration. - pub fn peek(&self, id: &str) -> Result, RepositoryError> { - let entity = self.inner.repo().get_with(id, ReadOpts::no_lock())?; - let Some(entity) = entity else { - return Ok(None); - }; - let snapshot = self.inner.repo().get_snapshot(id)?; - Ok(Some(hydrate_with_optional_snapshot::(entity, snapshot)?)) - } -} - -impl SnapshotAggregateRepository -where - R: GetAllWithOpts + SnapshotStore, - A: Snapshottable, -{ - /// Non-locking bulk read with snapshot-aware hydration. - pub fn peek_all(&self, ids: &[&str]) -> Result, RepositoryError> { - let entities = self.inner.repo().get_all_with(ids, ReadOpts::no_lock())?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - let snapshot = self.inner.repo().get_snapshot(entity.id())?; - aggregates.push(hydrate_with_optional_snapshot::(entity, snapshot)?); - } - Ok(aggregates) - } -} - -// ============================================================================ -// Outbox integration — delegate through inner AggregateRepository -// ============================================================================ - -impl SnapshotAggregateRepository -where - R: TransactionalCommit, - A: Snapshottable, -{ - /// Start an outbox commit chain, same as AggregateRepository. - pub fn outbox<'a>( - &'a self, - outbox: crate::outbox::OutboxMessage, - ) -> SnapshotOutboxCommit<'a, R, A> { - SnapshotOutboxCommit { - snap_repo: self, - outbox, - } - } -} - -/// Helper for chaining outbox + snapshot-aware commit. -pub struct SnapshotOutboxCommit<'a, R, A> { - snap_repo: &'a SnapshotAggregateRepository, - outbox: crate::outbox::OutboxMessage, -} - -impl<'a, R, A> SnapshotOutboxCommit<'a, R, A> -where - R: TransactionalCommit, - A: Snapshottable, -{ - pub fn commit(mut self, aggregate: &mut A) -> Result<(), RepositoryError> { - let snapshot = self.snap_repo.snapshot_record(aggregate)?; - let snapshot_version = snapshot.as_ref().map(|record| record.version); - let snapshots = snapshot.into_iter().map(SnapshotWrite::Save).collect(); - self.outbox.set_source(aggregate); - - let mut batch = CommitBatch { - entities: vec![aggregate.entity_mut()], - outbox_messages: Vec::new(), - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - }; - batch.outbox_messages.push(self.outbox); - self.snap_repo.inner.repo().commit_batch(batch)?; - - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - Ok(()) - } -} - #[cfg(test)] mod tests { use super::*; - use crate::{sourced, Aggregate, AggregateRepository, Entity, EventRecord}; - use std::cell::RefCell; + use crate::{sourced, Aggregate, EventRecord}; #[derive(Default)] struct TestAggregate { @@ -581,36 +348,65 @@ mod tests { #[derive(Default)] struct FailingSnapshotRepo { - saw_snapshot: RefCell, - } - - impl TransactionalCommit for FailingSnapshotRepo { - fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - if !batch.snapshots.is_empty() { - *self.saw_snapshot.borrow_mut() = true; - return Err(RepositoryError::Model("snapshot write failed".into())); + saw_snapshot: std::sync::atomic::AtomicBool, + } + + impl AsyncTransactionalCommit for FailingSnapshotRepo { + async fn commit_batch_async<'a>( + &'a self, + batch: AsyncCommitBatch<'a>, + ) -> Result<(), RepositoryError> { + { + if !batch.snapshots.is_empty() { + self.saw_snapshot + .store(true, std::sync::atomic::Ordering::SeqCst); + return Err(RepositoryError::Model("snapshot write failed".into())); + } + + for stream in batch.streams { + stream.entity.mark_committed(); + } + Ok(()) } + } + } - for entity in batch.entities { - entity.mark_committed(); + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; } - Ok(()) } } #[test] fn snapshot_batch_failure_leaves_aggregate_uncommitted() { let repo = FailingSnapshotRepo::default(); - let aggregate_repo = AggregateRepository::new(repo); - let snapshot_repo = SnapshotAggregateRepository::new(aggregate_repo, 1); + let aggregate_repo = AsyncAggregateRepository::new(repo); + let snapshot_repo = AsyncSnapshotAggregateRepository::new(aggregate_repo, 1); let mut aggregate = TestAggregate::default(); aggregate.touch().unwrap(); - let err = snapshot_repo.commit(&mut aggregate).unwrap_err(); + let err = block_on(snapshot_repo.commit(&mut aggregate)).unwrap_err(); assert_eq!(err, RepositoryError::Model("snapshot write failed".into())); - assert!(*snapshot_repo.repo().repo().saw_snapshot.borrow()); + assert!(snapshot_repo + .repo() + .repo() + .saw_snapshot + .load(std::sync::atomic::Ordering::SeqCst)); assert_eq!(aggregate.entity.committed_version(), 0); assert_eq!(aggregate.entity.snapshot_version(), 0); assert_eq!(aggregate.entity.new_events().len(), 1); diff --git a/src/snapshot/store.rs b/src/snapshot/store.rs index a06e96d90..79a8c6d0f 100644 --- a/src/snapshot/store.rs +++ b/src/snapshot/store.rs @@ -110,18 +110,3 @@ impl SnapshotRecord { && self.payload_codec_version == BITCODE_PAYLOAD_CODEC_VERSION } } - -/// Trait for ID-only snapshot persistence. One snapshot per aggregate ID (latest wins). -/// -/// Durable async repositories should prefer `AsyncSnapshotStore`, which keys -/// cache records by full `StreamIdentity`. -pub trait SnapshotStore: Send + Sync { - /// Load the latest snapshot for the given aggregate ID. - fn get_snapshot(&self, id: &str) -> Result, RepositoryError>; - - /// Save (or overwrite) the snapshot for the given aggregate ID. - fn save_snapshot(&self, record: SnapshotRecord) -> Result<(), RepositoryError>; - - /// Delete the snapshot for the given aggregate ID. Returns true if one existed. - fn delete_snapshot(&self, id: &str) -> Result; -} diff --git a/tests/bomberman/handlers/create_game.rs b/tests/bomberman/handlers/create_game.rs index 35911c0a1..f7669c7f1 100644 --- a/tests/bomberman/handlers/create_game.rs +++ b/tests/bomberman/handlers/create_game.rs @@ -1,16 +1,13 @@ -use sourced_rust::{ - Commit, ReadModelWritePlanStore, RelationalReadModelQueryStore, - SyncReadModelWritePlanCommitExt, TransactionalCommit, -}; +use sourced_rust::{AsyncReadModelWritePlanCommitExt, AsyncTransactionalCommit}; use super::shared::board_write_plan; use crate::domain::game_map::GameMap; use crate::error::GameError; use crate::views::BoardView; -pub fn create_game(repo: &R, game_id: &str, ascii_map: &str) -> Result +pub async fn create_game(repo: &R, game_id: &str, ascii_map: &str) -> Result where - R: Commit + TransactionalCommit + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncTransactionalCommit, { let (width, height, tiles, spawn_points) = GameMap::from_ascii(ascii_map); @@ -18,8 +15,9 @@ where map.create(game_id.into(), width, height, tiles.clone(), spawn_points)?; let board = BoardView::new(game_id, width, height, tiles); - repo.read_models_sync(board_write_plan(&board)?) - .commit_sync(&mut map)?; + repo.read_models(board_write_plan(&board)?) + .commit(&mut map) + .await?; Ok(map) } diff --git a/tests/bomberman/handlers/get_player.rs b/tests/bomberman/handlers/get_player.rs index f5b481c2a..31d046904 100644 --- a/tests/bomberman/handlers/get_player.rs +++ b/tests/bomberman/handlers/get_player.rs @@ -1,9 +1,11 @@ -use sourced_rust::{Get, GetAggregate}; +use sourced_rust::AsyncGetStream; +use super::shared::get_aggregate; use crate::domain::player::Player; use crate::error::GameError; -pub fn get_player(repo: &R, player_id: &str) -> Result { - repo.get_aggregate(&format!("player:{}", player_id))? +pub async fn get_player(repo: &R, player_id: &str) -> Result { + get_aggregate::(repo, &format!("player:{}", player_id)) + .await? .ok_or(GameError::PlayerNotFound(player_id.to_string())) } diff --git a/tests/bomberman/handlers/join_game.rs b/tests/bomberman/handlers/join_game.rs index e799ef853..a8127fdac 100644 --- a/tests/bomberman/handlers/join_game.rs +++ b/tests/bomberman/handlers/join_game.rs @@ -1,17 +1,17 @@ use sourced_rust::{ - Commit, Get, GetAggregate, ReadModelWritePlanStore, RelationalReadModelQueryStore, - SyncReadModelWritePlanCommitExt, TransactionalCommit, + AsyncGetStream, AsyncReadModelWritePlanCommitExt, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, }; use super::shared::{ - board_write_plan, build_board_from_aggregates, load_board, load_board_bombs, + board_write_plan, build_board_from_aggregates, get_aggregate, load_board, load_board_bombs, load_board_explosions, load_board_players, }; use crate::domain::game_map::GameMap; use crate::domain::player::Player; use crate::error::GameError; -pub fn join_game( +pub async fn join_game( repo: &R, player_id: &str, name: &str, @@ -19,10 +19,13 @@ pub fn join_game( spawn_index: usize, ) -> Result<(), GameError> where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - let map: GameMap = repo - .get_aggregate(game_id)? + let map: GameMap = get_aggregate::(repo, game_id) + .await? .ok_or(GameError::GameNotFound)?; let (sx, sy) = map @@ -34,12 +37,12 @@ where let mut player = Player::default(); player.join(format!("player:{}", player_id), name.into(), sx, sy)?; - let current_board = load_board(repo, game_id)?; - let mut all_players = load_board_players(repo, ¤t_board)?; + let current_board = load_board(repo, game_id).await?; + let mut all_players = load_board_players(repo, ¤t_board).await?; all_players.push(player.clone()); - let all_bombs = load_board_bombs(repo, ¤t_board)?; - let all_explosions = load_board_explosions(repo, ¤t_board)?; + let all_bombs = load_board_bombs(repo, ¤t_board).await?; + let all_explosions = load_board_explosions(repo, ¤t_board).await?; let board = build_board_from_aggregates( game_id, &map, @@ -50,8 +53,9 @@ where current_board.explosions_created, ); - repo.read_models_sync(board_write_plan(&board)?) - .commit_sync(&mut player)?; + repo.read_models(board_write_plan(&board)?) + .commit(&mut player) + .await?; Ok(()) } diff --git a/tests/bomberman/handlers/mod.rs b/tests/bomberman/handlers/mod.rs index 67b22469b..dfd1c0f02 100644 --- a/tests/bomberman/handlers/mod.rs +++ b/tests/bomberman/handlers/mod.rs @@ -11,4 +11,5 @@ pub use get_player::get_player; pub use join_game::join_game; pub use move_player::move_player; pub use place_bomb::place_bomb; +pub(crate) use shared::get_aggregate; pub use tick::tick; diff --git a/tests/bomberman/handlers/move_player.rs b/tests/bomberman/handlers/move_player.rs index 764a6ac81..c59aec606 100644 --- a/tests/bomberman/handlers/move_player.rs +++ b/tests/bomberman/handlers/move_player.rs @@ -1,10 +1,10 @@ use sourced_rust::{ - Aggregate, Commit, Get, GetAggregate, ReadModelWritePlanStore, RelationalReadModelQueryStore, - SyncReadModelWritePlanCommitExt, TransactionalCommit, + AsyncGetStream, AsyncReadModelWritePlanCommitExt, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, }; use super::shared::{ - board_write_plan, build_board_from_aggregates, load_board, load_board_bombs, + board_write_plan, build_board_from_aggregates, get_aggregate, load_board, load_board_bombs, load_board_explosions, load_board_players, }; use crate::domain::game_map::GameMap; @@ -12,21 +12,24 @@ use crate::domain::player::Player; use crate::domain::types::Direction; use crate::error::GameError; -pub fn move_player( +pub async fn move_player( repo: &R, player_id: &str, direction: Direction, game_id: &str, ) -> Result<(), GameError> where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - let mut map: GameMap = repo - .get_aggregate(game_id)? + let mut map: GameMap = get_aggregate::(repo, game_id) + .await? .ok_or(GameError::GameNotFound)?; - let mut player: Player = repo - .get_aggregate(&format!("player:{}", player_id))? + let mut player: Player = get_aggregate::(repo, &format!("player:{}", player_id)) + .await? .ok_or(GameError::PlayerNotFound(player_id.to_string()))?; if !player.alive { @@ -48,15 +51,16 @@ where player.apply_power_up(power_up)?; } - let current_board = load_board(repo, game_id)?; - let mut all_players: Vec = load_board_players(repo, ¤t_board)? + let current_board = load_board(repo, game_id).await?; + let mut all_players: Vec = load_board_players(repo, ¤t_board) + .await? .into_iter() .filter(|existing| existing.entity.id() != player.entity.id()) .collect(); all_players.push(player.clone()); - let all_bombs = load_board_bombs(repo, ¤t_board)?; - let all_explosions = load_board_explosions(repo, ¤t_board)?; + let all_bombs = load_board_bombs(repo, ¤t_board).await?; + let all_explosions = load_board_explosions(repo, ¤t_board).await?; let board = build_board_from_aggregates( game_id, &map, @@ -67,8 +71,11 @@ where current_board.explosions_created, ); - repo.read_models_sync(board_write_plan(&board)?) - .commit_many_sync(&mut [map.entity_mut(), player.entity_mut()])?; + repo.read_models(board_write_plan(&board)?) + .aggregate(&mut map) + .aggregate(&mut player) + .commit() + .await?; Ok(()) } diff --git a/tests/bomberman/handlers/place_bomb.rs b/tests/bomberman/handlers/place_bomb.rs index 3d07030f2..48af114cd 100644 --- a/tests/bomberman/handlers/place_bomb.rs +++ b/tests/bomberman/handlers/place_bomb.rs @@ -1,10 +1,10 @@ use sourced_rust::{ - Aggregate, Commit, Get, GetAggregate, ReadModelWritePlanStore, RelationalReadModelQueryStore, - SyncReadModelWritePlanCommitExt, TransactionalCommit, + AsyncGetStream, AsyncReadModelWritePlanCommitExt, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, }; use super::shared::{ - board_write_plan, build_board_from_aggregates, load_board, load_board_bombs, + board_write_plan, build_board_from_aggregates, get_aggregate, load_board, load_board_bombs, load_board_explosions, load_board_players, }; use crate::domain::bomb::Bomb; @@ -12,16 +12,19 @@ use crate::domain::game_map::GameMap; use crate::domain::player::Player; use crate::error::GameError; -pub fn place_bomb(repo: &R, player_id: &str, game_id: &str) -> Result<(), GameError> +pub async fn place_bomb(repo: &R, player_id: &str, game_id: &str) -> Result<(), GameError> where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - let map: GameMap = repo - .get_aggregate(game_id)? + let map: GameMap = get_aggregate::(repo, game_id) + .await? .ok_or(GameError::GameNotFound)?; - let mut player: Player = repo - .get_aggregate(&format!("player:{}", player_id))? + let mut player: Player = get_aggregate::(repo, &format!("player:{}", player_id)) + .await? .ok_or(GameError::PlayerNotFound(player_id.to_string()))?; if !player.alive { @@ -44,20 +47,22 @@ where player.blast_radius, )?; - let current_board = load_board(repo, game_id)?; - let mut all_players: Vec = load_board_players(repo, ¤t_board)? + let current_board = load_board(repo, game_id).await?; + let mut all_players: Vec = load_board_players(repo, ¤t_board) + .await? .into_iter() .filter(|existing| existing.entity.id() != player.entity.id()) .collect(); all_players.push(player.clone()); - let mut all_bombs: Vec = load_board_bombs(repo, ¤t_board)? + let mut all_bombs: Vec = load_board_bombs(repo, ¤t_board) + .await? .into_iter() .filter(|existing| existing.entity.id() != bomb.entity.id()) .collect(); all_bombs.push(bomb.clone()); - let all_explosions = load_board_explosions(repo, ¤t_board)?; + let all_explosions = load_board_explosions(repo, ¤t_board).await?; let board = build_board_from_aggregates( game_id, &map, @@ -68,8 +73,11 @@ where current_board.explosions_created, ); - repo.read_models_sync(board_write_plan(&board)?) - .commit_many_sync(&mut [player.entity_mut(), bomb.entity_mut()])?; + repo.read_models(board_write_plan(&board)?) + .aggregate(&mut player) + .aggregate(&mut bomb) + .commit() + .await?; Ok(()) } diff --git a/tests/bomberman/handlers/shared.rs b/tests/bomberman/handlers/shared.rs index adbea314d..7a917c556 100644 --- a/tests/bomberman/handlers/shared.rs +++ b/tests/bomberman/handlers/shared.rs @@ -1,6 +1,7 @@ use sourced_rust::{ - Aggregate, Get, GetAggregate, ReadModelWorkspaceExt, ReadModelWritePlanBuilder, - ReadModelWritePlanStore, RelationalReadModelQueryStore, RepositoryError, RowKey, RowValue, + hydrate, Aggregate, AsyncGetStream, AsyncReadModelWorkspaceExt, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, ReadModelWritePlanBuilder, RepositoryError, RowKey, + RowValue, StreamIdentity, }; use crate::domain::bomb::Bomb; @@ -20,51 +21,66 @@ pub(crate) fn board_write_plan(board: &BoardView) -> Result(repo: &R, game_id: &str) -> Result +pub(crate) async fn load_board(repo: &R, game_id: &str) -> Result where - R: ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncReadModelWritePlanStore + AsyncRelationalReadModelQueryStore, { - repo.workspace() - .load::(board_key(game_id)) + repo.workspace_async() + .load_async::(board_key(game_id)) .one() + .await .map_err(RepositoryError::from)? .map(|board| board.data) .ok_or(GameError::GameNotFound) } -fn load_indexed_aggregates(repo: &R, ids: I) -> Result, GameError> +/// Load and hydrate a single aggregate by its raw id, keyed under the +/// aggregate's own stream type. Works through `&R` (handlers borrow the repo). +pub(crate) async fn get_aggregate(repo: &R, id: &str) -> Result, GameError> where - R: Get, - A: Aggregate, + R: AsyncGetStream, + A: Aggregate + Send, +{ + let identity = StreamIdentity::new(A::aggregate_type(), id).map_err(GameError::Repository)?; + let Some(entity) = repo.get_stream(&identity).await? else { + return Ok(None); + }; + Ok(Some(hydrate::(entity).map_err(GameError::Repository)?)) +} + +async fn load_indexed_aggregates(repo: &R, ids: I) -> Result, GameError> +where + R: AsyncGetStream, + A: Aggregate + Send, I: IntoIterator, I::Item: AsRef, { let mut results = Vec::new(); for id in ids { let id = id.as_ref(); - let aggregate = repo - .get_aggregate(id)? + let aggregate = get_aggregate::(repo, id) + .await? .ok_or_else(|| RepositoryError::NotFound { id: id.to_string() })?; results.push(aggregate); } Ok(results) } -pub(crate) fn load_board_players( +pub(crate) async fn load_board_players( repo: &R, board: &BoardView, ) -> Result, GameError> { - load_indexed_aggregates(repo, board.players.iter().map(|player| player.id.as_str())) + load_indexed_aggregates(repo, board.players.iter().map(|player| player.id.as_str())).await } -pub(crate) fn load_board_bombs( +pub(crate) async fn load_board_bombs( repo: &R, board: &BoardView, ) -> Result, GameError> { - load_indexed_aggregates(repo, board.bombs.iter().map(|bomb| bomb.id.as_str())) + load_indexed_aggregates(repo, board.bombs.iter().map(|bomb| bomb.id.as_str())).await } -pub(crate) fn load_board_explosions( +pub(crate) async fn load_board_explosions( repo: &R, board: &BoardView, ) -> Result, GameError> { @@ -75,6 +91,7 @@ pub(crate) fn load_board_explosions( .iter() .map(|explosion| explosion.id.as_str()), ) + .await } pub(crate) fn build_board_from_aggregates( diff --git a/tests/bomberman/handlers/tick.rs b/tests/bomberman/handlers/tick.rs index c701bc32e..ec8ae7691 100644 --- a/tests/bomberman/handlers/tick.rs +++ b/tests/bomberman/handlers/tick.rs @@ -1,11 +1,11 @@ use sourced_rust::{ - Aggregate, Commit, Get, GetAggregate, OutboxMessage, ReadModelWritePlanStore, - RelationalReadModelQueryStore, RepositoryError, SyncReadModelWritePlanCommitExt, - TransactionalCommit, + Aggregate, AsyncCommitBatch, AsyncGetStream, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncStreamWrite, AsyncTransactionalCommit, OutboxMessage, + RepositoryError, StreamIdentity, }; use super::shared::{ - board_write_plan, build_board_from_aggregates, load_board, load_board_bombs, + board_write_plan, build_board_from_aggregates, get_aggregate, load_board, load_board_bombs, load_board_explosions, load_board_players, }; use crate::domain::bomb::Bomb; @@ -37,18 +37,21 @@ struct KillAttribution { bomb_owner: String, } -pub fn tick(repo: &R, game_id: &str) -> Result +pub async fn tick(repo: &R, game_id: &str) -> Result where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - let mut map: GameMap = repo - .get_aggregate(game_id)? + let mut map: GameMap = get_aggregate::(repo, game_id) + .await? .ok_or(GameError::GameNotFound)?; - let board = load_board(repo, game_id)?; - let mut bombs = load_board_bombs(repo, &board)?; - let mut players = load_board_players(repo, &board)?; - let mut explosions = load_board_explosions(repo, &board)?; + let board = load_board(repo, game_id).await?; + let mut bombs = load_board_bombs(repo, &board).await?; + let mut players = load_board_players(repo, &board).await?; + let mut explosions = load_board_explosions(repo, &board).await?; let bombs_ticked = bombs.len(); for bomb in &mut bombs { @@ -181,7 +184,11 @@ where explosion_counter, ); - let mut builder = repo.read_models_sync(board_write_plan(&board)?); + let read_model_plan = board_write_plan(&board)? + .into_write_plan() + .map_err(RepositoryError::from)?; + + let mut outbox_messages = Vec::new(); for killed_id in &saga.players_killed { let attribution = kill_attributions .iter() @@ -204,22 +211,37 @@ where ))) })?, )?; - builder = builder.outbox_sync(outbox); + outbox_messages.push(outbox); } - let mut entities: Vec<&mut sourced_rust::Entity> = Vec::new(); - entities.push(map.entity_mut()); + // Stage every touched aggregate stream under its own type's stream identity. + let mut streams: Vec> = Vec::new(); + let map_identity = StreamIdentity::new(GameMap::aggregate_type(), map.entity.id()) + .map_err(GameError::Repository)?; + streams.push(AsyncStreamWrite::new(map_identity, map.entity_mut())); for player in &mut players { - entities.push(player.entity_mut()); + let identity = StreamIdentity::new(Player::aggregate_type(), player.entity.id()) + .map_err(GameError::Repository)?; + streams.push(AsyncStreamWrite::new(identity, player.entity_mut())); } for bomb in &mut bombs { - entities.push(bomb.entity_mut()); + let identity = StreamIdentity::new(Bomb::aggregate_type(), bomb.entity.id()) + .map_err(GameError::Repository)?; + streams.push(AsyncStreamWrite::new(identity, bomb.entity_mut())); } for explosion in &mut explosions { - entities.push(explosion.entity_mut()); + let identity = StreamIdentity::new(Explosion::aggregate_type(), explosion.entity.id()) + .map_err(GameError::Repository)?; + streams.push(AsyncStreamWrite::new(identity, explosion.entity_mut())); } - entities.push(saga.entity_mut()); - builder.commit_many_sync(&mut entities)?; + let saga_identity = StreamIdentity::new(TickSaga::aggregate_type(), saga.entity.id()) + .map_err(GameError::Repository)?; + streams.push(AsyncStreamWrite::new(saga_identity, saga.entity_mut())); + + let mut batch = AsyncCommitBatch::new(streams); + batch.outbox_messages = outbox_messages; + batch.read_model_plans = vec![read_model_plan]; + repo.commit_batch_async(batch).await?; Ok(saga) } diff --git a/tests/bomberman/main.rs b/tests/bomberman/main.rs index 7ecfb87d7..fc2cf20e0 100644 --- a/tests/bomberman/main.rs +++ b/tests/bomberman/main.rs @@ -2,10 +2,9 @@ //! //! A 4-player Bomberman demonstrates: //! - Single aggregate + atomic read model commit (player join/move) -//! - Multi-aggregate atomic commit via `commit_many` (bomb placement, explosion) +//! - Multi-aggregate atomic commit via the staged commit builder (bomb placement) //! - Aggregate lifecycle: bomb created -> ticked -> exploded -> explosion expands //! - In-process orchestration saga (tick resolves explosions across aggregates) -//! - Contested resources via QueuedRepository (concurrent bomb placement) //! - Composite read model (BoardView from map + players + bombs + explosions) //! - Outbox events ("PlayerKilled" on death) //! - Guard conditions (can't move when dead, can't bomb at max) @@ -35,26 +34,26 @@ const SMALL_MAP: &str = "\ // Pattern: Single aggregate + read model commit, terrain validation // ============================================================================ -#[test] -fn game_setup_and_movement() { +#[tokio::test] +async fn game_setup_and_movement() { let repo = HashMapRepository::new(); - let game = Game::new(&repo, "game-1", SMALL_MAP).unwrap(); + let game = Game::new(&repo, "game-1", SMALL_MAP).await.unwrap(); let p1 = game.sim("p1", "Alice"); let p2 = game.sim("p2", "Bob"); - p1.join(0).unwrap(); - p2.join(1).unwrap(); + p1.join(0).await.unwrap(); + p2.join(1).await.unwrap(); // P1 starts at spawn 0 (1,1), move south - p1.move_dir(Direction::South).unwrap(); + p1.move_dir(Direction::South).await.unwrap(); // Try to move into a wall — should fail - let result = p1.move_dir(Direction::West); + let result = p1.move_dir(Direction::West).await; assert!(result.is_err()); // Verify board view - let board = game.board().unwrap(); + let board = game.board().await.unwrap(); assert_eq!(board.data.players.len(), 2); let alice = board @@ -73,13 +72,15 @@ fn game_setup_and_movement() { assert!(bob.alive); } -#[test] -fn invalid_spawn_index_returns_error() { +#[tokio::test] +async fn invalid_spawn_index_returns_error() { let repo = HashMapRepository::new(); - let game = Game::new(&repo, "game-invalid-spawn", SMALL_MAP).unwrap(); + let game = Game::new(&repo, "game-invalid-spawn", SMALL_MAP) + .await + .unwrap(); let player = game.sim("p1", "Alice"); - let result = player.join(99); + let result = player.join(99).await; assert!(matches!( result, @@ -92,48 +93,48 @@ fn invalid_spawn_index_returns_error() { // Pattern: Aggregate lifecycle (bomb created -> ticked -> exploded -> expansion) // ============================================================================ -#[test] -fn bomb_destroys_blocks() { +#[tokio::test] +async fn bomb_destroys_blocks() { let repo = HashMapRepository::new(); - let game = Game::new(&repo, "game-2", SMALL_MAP).unwrap(); + let game = Game::new(&repo, "game-2", SMALL_MAP).await.unwrap(); let p1 = game.sim("p1", "Alice"); - p1.join(0).unwrap(); + p1.join(0).await.unwrap(); // Alice at (1,1). Move to (2,1) to be near block at (3,1). - p1.move_dir(Direction::East).unwrap(); + p1.move_dir(Direction::East).await.unwrap(); // Place bomb at (2,1) — blast radius 2 east reaches block at (3,1) at ring 1 - p1.place_bomb().unwrap(); + p1.place_bomb().await.unwrap(); // Escape: go west to (1,1) then south to (1,2) - p1.move_dir(Direction::West).unwrap(); // (1,1) - p1.move_dir(Direction::South).unwrap(); // (1,2) — safe from bomb at (2,1) + p1.move_dir(Direction::West).await.unwrap(); // (1,1) + p1.move_dir(Direction::South).await.unwrap(); // (1,2) — safe from bomb at (2,1) // Verify bomb is on the board - let board = game.board().unwrap(); + let board = game.board().await.unwrap(); assert_eq!(board.data.bombs.len(), 1); // Tick 3 times: bomb timer 3 -> 2 -> 1 -> 0, detonates, center (2,1) active - game.tick().unwrap(); - game.tick().unwrap(); - game.tick().unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); // Tick 4: explosion expands to ring 1 → (3,1) block destroyed - let tick_result = game.tick().unwrap(); + let tick_result = game.tick().await.unwrap(); // Block should be destroyed this tick assert!(!tick_result.blocks_destroyed.is_empty()); // Verify bomb gone from board - let board = game.board().unwrap(); + let board = game.board().await.unwrap(); assert_eq!(board.data.bombs.len(), 0); // Alice should still be alive (moved away) - assert!(p1.is_alive().unwrap()); + assert!(p1.is_alive().await.unwrap()); // Verify bomb returned to player - let player = p1.player().unwrap(); + let player = p1.player().await.unwrap(); assert_eq!(player.active_bombs, 0); } @@ -142,8 +143,8 @@ fn bomb_destroys_blocks() { // Pattern: Multi-aggregate coordination + outbox event // ============================================================================ -#[test] -fn player_killed_by_bomb() { +#[tokio::test] +async fn player_killed_by_bomb() { // Use a tall open map so players can retreat far from blast let repo2 = HashMapRepository::new(); let open_map = "\ @@ -155,51 +156,51 @@ fn player_killed_by_bomb() { # # # 2# ###########"; - let game2 = Game::new(&repo2, "game-3b", open_map).unwrap(); + let game2 = Game::new(&repo2, "game-3b", open_map).await.unwrap(); let alice = game2.sim("p1", "Alice"); let bob = game2.sim("p2", "Bob"); - alice.join(0).unwrap(); // (1,1) - bob.join(1).unwrap(); // (9,6) + alice.join(0).await.unwrap(); // (1,1) + bob.join(1).await.unwrap(); // (9,6) // Move Bob to (5,1) via north then west - bob.move_dir(Direction::North).unwrap(); // (9,5) - bob.move_dir(Direction::North).unwrap(); // (9,4) - bob.move_dir(Direction::North).unwrap(); // (9,3) - bob.move_dir(Direction::North).unwrap(); // (9,2) - bob.move_dir(Direction::North).unwrap(); // (9,1) - bob.move_dir(Direction::West).unwrap(); // (8,1) - bob.move_dir(Direction::West).unwrap(); // (7,1) - bob.move_dir(Direction::West).unwrap(); // (6,1) - bob.move_dir(Direction::West).unwrap(); // (5,1) + bob.move_dir(Direction::North).await.unwrap(); // (9,5) + bob.move_dir(Direction::North).await.unwrap(); // (9,4) + bob.move_dir(Direction::North).await.unwrap(); // (9,3) + bob.move_dir(Direction::North).await.unwrap(); // (9,2) + bob.move_dir(Direction::North).await.unwrap(); // (9,1) + bob.move_dir(Direction::West).await.unwrap(); // (8,1) + bob.move_dir(Direction::West).await.unwrap(); // (7,1) + bob.move_dir(Direction::West).await.unwrap(); // (6,1) + bob.move_dir(Direction::West).await.unwrap(); // (5,1) // Alice moves east to (3,1) - alice.move_dir(Direction::East).unwrap(); // (2,1) - alice.move_dir(Direction::East).unwrap(); // (3,1) + alice.move_dir(Direction::East).await.unwrap(); // (2,1) + alice.move_dir(Direction::East).await.unwrap(); // (3,1) // Alice places bomb at (3,1), blast radius 2 east: ring 1=(4,1), ring 2=(5,1) - alice.place_bomb().unwrap(); + alice.place_bomb().await.unwrap(); // Alice retreats south 3 cells (blast south goes (3,2),(3,3), so (3,4) is safe) - alice.move_dir(Direction::South).unwrap(); // (3,2) - alice.move_dir(Direction::South).unwrap(); // (3,3) - alice.move_dir(Direction::South).unwrap(); // (3,4) + alice.move_dir(Direction::South).await.unwrap(); // (3,2) + alice.move_dir(Direction::South).await.unwrap(); // (3,3) + alice.move_dir(Direction::South).await.unwrap(); // (3,4) // Tick 3 times to detonate: timer 3→2→1→0, bomb detonates, center (3,1) active - game2.tick().unwrap(); - game2.tick().unwrap(); - game2.tick().unwrap(); + game2.tick().await.unwrap(); + game2.tick().await.unwrap(); + game2.tick().await.unwrap(); // Tick 4: expand to ring 1 → (4,1) — Bob not here - game2.tick().unwrap(); + game2.tick().await.unwrap(); // Tick 5: expand to ring 2 → (5,1) — Bob killed! - let tick_result = game2.tick().unwrap(); + let tick_result = game2.tick().await.unwrap(); // Verify Bob was killed - assert!(!bob.is_alive().unwrap()); - assert!(alice.is_alive().unwrap()); + assert!(!bob.is_alive().await.unwrap()); + assert!(alice.is_alive().await.unwrap()); // Verify kill recorded on saga assert!(tick_result @@ -221,8 +222,8 @@ fn player_killed_by_bomb() { // Pattern: In-process orchestration saga (expanding explosion triggers chain) // ============================================================================ -#[test] -fn chain_reaction() { +#[tokio::test] +async fn chain_reaction() { let repo = HashMapRepository::new(); // Tall map so players can retreat far enough from blast radius 2 let tall_map = "\ @@ -235,50 +236,50 @@ fn chain_reaction() { # # # 2# ###########"; - let game = Game::new(&repo, "game-4", tall_map).unwrap(); + let game = Game::new(&repo, "game-4", tall_map).await.unwrap(); let p1 = game.sim("p1", "Alice"); let p2 = game.sim("p2", "Bob"); - p1.join(0).unwrap(); // (1,1) - p2.join(1).unwrap(); // (9,7) + p1.join(0).await.unwrap(); // (1,1) + p2.join(1).await.unwrap(); // (9,7) // Alice moves to (3,1), places bomb, then retreats south far enough - p1.move_dir(Direction::East).unwrap(); // (2,1) - p1.move_dir(Direction::East).unwrap(); // (3,1) - p1.place_bomb().unwrap(); // bomb at (3,1), radius 2 + p1.move_dir(Direction::East).await.unwrap(); // (2,1) + p1.move_dir(Direction::East).await.unwrap(); // (3,1) + p1.place_bomb().await.unwrap(); // bomb at (3,1), radius 2 // Alice retreats south 3 cells (blast goes south only 2) - p1.move_dir(Direction::South).unwrap(); // (3,2) - p1.move_dir(Direction::South).unwrap(); // (3,3) - p1.move_dir(Direction::South).unwrap(); // (3,4) + p1.move_dir(Direction::South).await.unwrap(); // (3,2) + p1.move_dir(Direction::South).await.unwrap(); // (3,3) + p1.move_dir(Direction::South).await.unwrap(); // (3,4) // Tick twice so Alice's bomb timer goes from 3→2→1 - game.tick().unwrap(); - game.tick().unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); // Now Bob places bomb at (5,1) — 2 cells east of Alice's bomb // Bob's bomb will have timer=3, Alice's bomb has timer=1 // Alice's bomb detonates next tick, expanding blast reaches (5,1) at ring 2 - bob_move_to_position(&game, "p2", "Bob", 9, 7, 5, 1); - p2.place_bomb().unwrap(); // bomb at (5,1), radius 2 - p2.move_dir(Direction::South).unwrap(); // (5,2) - p2.move_dir(Direction::South).unwrap(); // (5,3) - p2.move_dir(Direction::South).unwrap(); // (5,4) + bob_move_to_position(&game, "p2", "Bob", 9, 7, 5, 1).await; + p2.place_bomb().await.unwrap(); // bomb at (5,1), radius 2 + p2.move_dir(Direction::South).await.unwrap(); // (5,2) + p2.move_dir(Direction::South).await.unwrap(); // (5,3) + p2.move_dir(Direction::South).await.unwrap(); // (5,4) // Tick 3: Alice's bomb timer 1→0 → detonates. Bob's bomb timer 3→2. // Alice's explosion: center (3,1) active. - game.tick().unwrap(); + game.tick().await.unwrap(); // Tick 4: Alice's explosion expands to ring 1: (2,1),(4,1),(3,0)=wall blocked,(3,2). // Bob's bomb timer 2→1. No chain yet. - game.tick().unwrap(); + game.tick().await.unwrap(); // Tick 5: Alice's explosion expands to ring 2: (1,1),(5,1),(3,3). // (5,1) hits Bob's bomb → chain detonation! Bob's bomb marked ticks_remaining=0. // Bob's bomb timer was 1→0 from ticking, BUT the chain mark also sets it to 0. // Either way, Bob's bomb detonates in Phase B. Bob's explosion center (5,1) active. - let tick_result = game.tick().unwrap(); + let tick_result = game.tick().await.unwrap(); // Both bombs should have detonated — at least 2 detonations total across all ticks // The chain detonation should be recorded @@ -290,18 +291,12 @@ fn chain_reaction() { ); // Both players should be alive (they retreated far enough) - assert!(p1.is_alive().unwrap()); - assert!(p2.is_alive().unwrap()); + assert!(p1.is_alive().await.unwrap()); + assert!(p2.is_alive().await.unwrap()); } /// Helper to move a player step by step to a target position via simple pathfinding. -fn bob_move_to_position< - R: sourced_rust::Commit - + sourced_rust::TransactionalCommit - + sourced_rust::Get - + sourced_rust::ReadModelWritePlanStore - + sourced_rust::RelationalReadModelQueryStore, ->( +async fn bob_move_to_position( game: &Game<'_, R>, id: &str, _name: &str, @@ -309,7 +304,12 @@ fn bob_move_to_position< from_y: i32, to_x: i32, to_y: i32, -) { +) where + R: sourced_rust::AsyncGetStream + + sourced_rust::AsyncTransactionalCommit + + sourced_rust::AsyncReadModelWritePlanStore + + sourced_rust::AsyncRelationalReadModelQueryStore, +{ let sim = game.sim(id, _name); let mut cx = from_x; let mut cy = from_y; @@ -317,19 +317,19 @@ fn bob_move_to_position< // Move north/south first, then east/west while cy != to_y { if cy > to_y { - sim.move_dir(Direction::North).unwrap(); + sim.move_dir(Direction::North).await.unwrap(); cy -= 1; } else { - sim.move_dir(Direction::South).unwrap(); + sim.move_dir(Direction::South).await.unwrap(); cy += 1; } } while cx != to_x { if cx > to_x { - sim.move_dir(Direction::West).unwrap(); + sim.move_dir(Direction::West).await.unwrap(); cx -= 1; } else { - sim.move_dir(Direction::East).unwrap(); + sim.move_dir(Direction::East).await.unwrap(); cx += 1; } } @@ -340,13 +340,12 @@ fn bob_move_to_position< // Pattern: Contested resource — both players place bombs, verify atomic commits // ============================================================================ -#[test] -fn concurrent_bomb_placement() { - use sourced_rust::{ReadModelWorkspaceExt, RowKey, RowValue}; +#[tokio::test] +async fn concurrent_bomb_placement() { + use sourced_rust::{AsyncReadModelWorkspaceExt, RowKey, RowValue}; use std::sync::Arc; - use std::thread; - // HashMapRepository uses Arc> internally, safe to share across threads + // HashMapRepository uses Arc> internally, safe to share across tasks let repo = Arc::new(HashMapRepository::new()); let open_map = "\ ####### @@ -355,48 +354,64 @@ fn concurrent_bomb_placement() { #3 4# #######"; - // Create game and join players on main thread - handlers::create_game(&*repo, "game-5", open_map).unwrap(); - handlers::join_game(&*repo, "p1", "Alice", "game-5", 0).unwrap(); - handlers::join_game(&*repo, "p2", "Bob", "game-5", 1).unwrap(); + // Create game and join players first + handlers::create_game(&*repo, "game-5", open_map) + .await + .unwrap(); + handlers::join_game(&*repo, "p1", "Alice", "game-5", 0) + .await + .unwrap(); + handlers::join_game(&*repo, "p2", "Bob", "game-5", 1) + .await + .unwrap(); let repo2 = repo.clone(); let repo3 = repo.clone(); - // Two threads place bombs concurrently - let t1 = thread::spawn(move || { - handlers::place_bomb(&*repo2, "p1", "game-5").unwrap(); - }); - - let t2 = thread::spawn(move || { - handlers::place_bomb(&*repo3, "p2", "game-5").unwrap(); - }); - - t1.join().unwrap(); - t2.join().unwrap(); + // Two futures place bombs concurrently against the shared repository. + let (r1, r2) = tokio::join!( + handlers::place_bomb(&*repo2, "p1", "game-5"), + handlers::place_bomb(&*repo3, "p2", "game-5"), + ); + r1.unwrap(); + r2.unwrap(); // Verify both bombs placed let _board = repo - .workspace() - .load::(RowKey::new([( + .workspace_async() + .load_async::(RowKey::new([( "game_id", RowValue::String("game-5".into()), )])) .one() + .await .unwrap() .unwrap(); - // Board may show 1 or 2 bombs depending on which thread's board view won the race, + // Board may show 1 or 2 bombs depending on which task's board view won the race, // but both bomb aggregates should exist in the repo. - use sourced_rust::GetAggregate; - let alice: domain::player::Player = repo.get_aggregate("player:p1").unwrap().unwrap(); + let alice: domain::player::Player = + handlers::get_aggregate::<_, domain::player::Player>(&*repo, "player:p1") + .await + .unwrap() + .unwrap(); assert_eq!(alice.active_bombs, 1); - let bob: domain::player::Player = repo.get_aggregate("player:p2").unwrap().unwrap(); + let bob: domain::player::Player = + handlers::get_aggregate::<_, domain::player::Player>(&*repo, "player:p2") + .await + .unwrap() + .unwrap(); assert_eq!(bob.active_bombs, 1); // Both bomb entities should be in the repo - let bomb1: Option = repo.get_aggregate("bomb:p1:1").unwrap(); - let bomb2: Option = repo.get_aggregate("bomb:p2:1").unwrap(); + let bomb1: Option = + handlers::get_aggregate::<_, domain::bomb::Bomb>(&*repo, "bomb:p1:1") + .await + .unwrap(); + let bomb2: Option = + handlers::get_aggregate::<_, domain::bomb::Bomb>(&*repo, "bomb:p2:1") + .await + .unwrap(); assert!(bomb1.is_some(), "Alice's bomb should exist"); assert!(bomb2.is_some(), "Bob's bomb should exist"); } @@ -406,8 +421,8 @@ fn concurrent_bomb_placement() { // Pattern: End-to-end composition, game lifecycle // ============================================================================ -#[test] -fn full_game_to_winner() { +#[tokio::test] +async fn full_game_to_winner() { let repo = HashMapRepository::new(); // Larger arena to allow proper retreat from blast radius 2 let arena = "\ @@ -425,7 +440,7 @@ fn full_game_to_winner() { #3 4# #############"; - let game = Game::new(&repo, "game-6", arena).unwrap(); + let game = Game::new(&repo, "game-6", arena).await.unwrap(); let p1 = game.sim("p1", "Alice"); let p2 = game.sim("p2", "Bob"); @@ -433,101 +448,101 @@ fn full_game_to_winner() { let p4 = game.sim("p4", "Diana"); // All players join - p1.join(0).unwrap(); // (1,1) - p2.join(1).unwrap(); // (11,1) - p3.join(2).unwrap(); // (1,11) - p4.join(3).unwrap(); // (11,11) + p1.join(0).await.unwrap(); // (1,1) + p2.join(1).await.unwrap(); // (11,1) + p3.join(2).await.unwrap(); // (1,11) + p4.join(3).await.unwrap(); // (11,11) // Verify initial board - let board = game.board().unwrap(); + let board = game.board().await.unwrap(); assert_eq!(board.data.players.len(), 4); assert!(board.data.players.iter().all(|p| p.alive)); // --- Round 1: Alice traps Bob --- // Alice moves east to (9,1), places bomb. Blast east: ring 1=(10,1), ring 2=(11,1) hits Bob. - p1.move_dir(Direction::East).unwrap(); // (2,1) - p1.move_dir(Direction::East).unwrap(); // (3,1) - p1.move_dir(Direction::East).unwrap(); // (4,1) - p1.move_dir(Direction::East).unwrap(); // (5,1) - p1.move_dir(Direction::East).unwrap(); // (6,1) - p1.move_dir(Direction::East).unwrap(); // (7,1) - p1.move_dir(Direction::East).unwrap(); // (8,1) - p1.move_dir(Direction::East).unwrap(); // (9,1) + p1.move_dir(Direction::East).await.unwrap(); // (2,1) + p1.move_dir(Direction::East).await.unwrap(); // (3,1) + p1.move_dir(Direction::East).await.unwrap(); // (4,1) + p1.move_dir(Direction::East).await.unwrap(); // (5,1) + p1.move_dir(Direction::East).await.unwrap(); // (6,1) + p1.move_dir(Direction::East).await.unwrap(); // (7,1) + p1.move_dir(Direction::East).await.unwrap(); // (8,1) + p1.move_dir(Direction::East).await.unwrap(); // (9,1) - p1.place_bomb().unwrap(); // bomb at (9,1) + p1.place_bomb().await.unwrap(); // bomb at (9,1) // Alice retreats south 3 cells (blast south goes (9,2),(9,3)) - p1.move_dir(Direction::South).unwrap(); // (9,2) - p1.move_dir(Direction::South).unwrap(); // (9,3) - p1.move_dir(Direction::South).unwrap(); // (9,4) — safe + p1.move_dir(Direction::South).await.unwrap(); // (9,2) + p1.move_dir(Direction::South).await.unwrap(); // (9,3) + p1.move_dir(Direction::South).await.unwrap(); // (9,4) — safe // 3 ticks to detonate + 2 ticks for ring 2 expansion = 5 ticks - game.tick().unwrap(); - game.tick().unwrap(); - game.tick().unwrap(); // detonates, center active - game.tick().unwrap(); // ring 1 - let r1 = game.tick().unwrap(); // ring 2 → Bob killed + game.tick().await.unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); // detonates, center active + game.tick().await.unwrap(); // ring 1 + let r1 = game.tick().await.unwrap(); // ring 2 → Bob killed assert!( - !p2.is_alive().unwrap(), + !p2.is_alive().await.unwrap(), "Bob should be dead from Alice's bomb" ); assert!(r1.players_killed.contains(&"player:p2".to_string())); // --- Round 2: Charlie traps Diana --- // Charlie at (1,11), Diana at (11,11) - p3.move_dir(Direction::East).unwrap(); // (2,11) - p3.move_dir(Direction::East).unwrap(); // (3,11) - p3.move_dir(Direction::East).unwrap(); // (4,11) - p3.move_dir(Direction::East).unwrap(); // (5,11) - p3.move_dir(Direction::East).unwrap(); // (6,11) - p3.move_dir(Direction::East).unwrap(); // (7,11) - p3.move_dir(Direction::East).unwrap(); // (8,11) - p3.move_dir(Direction::East).unwrap(); // (9,11) + p3.move_dir(Direction::East).await.unwrap(); // (2,11) + p3.move_dir(Direction::East).await.unwrap(); // (3,11) + p3.move_dir(Direction::East).await.unwrap(); // (4,11) + p3.move_dir(Direction::East).await.unwrap(); // (5,11) + p3.move_dir(Direction::East).await.unwrap(); // (6,11) + p3.move_dir(Direction::East).await.unwrap(); // (7,11) + p3.move_dir(Direction::East).await.unwrap(); // (8,11) + p3.move_dir(Direction::East).await.unwrap(); // (9,11) - p3.place_bomb().unwrap(); // bomb at (9,11), blast east: ring 2=(11,11) hits Diana + p3.place_bomb().await.unwrap(); // bomb at (9,11), blast east: ring 2=(11,11) hits Diana // Charlie retreats north 3 cells - p3.move_dir(Direction::North).unwrap(); // (9,10) - p3.move_dir(Direction::North).unwrap(); // (9,9) - p3.move_dir(Direction::North).unwrap(); // (9,8) — safe + p3.move_dir(Direction::North).await.unwrap(); // (9,10) + p3.move_dir(Direction::North).await.unwrap(); // (9,9) + p3.move_dir(Direction::North).await.unwrap(); // (9,8) — safe // 5 ticks for detonation + expansion to ring 2 - game.tick().unwrap(); - game.tick().unwrap(); - game.tick().unwrap(); // detonates - game.tick().unwrap(); // ring 1 - let r2 = game.tick().unwrap(); // ring 2 → Diana killed + game.tick().await.unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); // detonates + game.tick().await.unwrap(); // ring 1 + let r2 = game.tick().await.unwrap(); // ring 2 → Diana killed - assert!(!p4.is_alive().unwrap(), "Diana should be dead"); + assert!(!p4.is_alive().await.unwrap(), "Diana should be dead"); assert!(r2.players_killed.contains(&"player:p4".to_string())); // --- Round 3: Alice finishes Charlie --- // Alice at (9,4), Charlie at (9,8) - p1.move_dir(Direction::South).unwrap(); // (9,5) - p1.move_dir(Direction::South).unwrap(); // (9,6) + p1.move_dir(Direction::South).await.unwrap(); // (9,5) + p1.move_dir(Direction::South).await.unwrap(); // (9,6) // Place bomb at (9,6), blast south: ring 1=(9,7), ring 2=(9,8) hits Charlie - p1.place_bomb().unwrap(); + p1.place_bomb().await.unwrap(); // Alice retreats north 3 cells - p1.move_dir(Direction::North).unwrap(); // (9,5) - p1.move_dir(Direction::North).unwrap(); // (9,4) - p1.move_dir(Direction::North).unwrap(); // (9,3) — safe + p1.move_dir(Direction::North).await.unwrap(); // (9,5) + p1.move_dir(Direction::North).await.unwrap(); // (9,4) + p1.move_dir(Direction::North).await.unwrap(); // (9,3) — safe // 5 ticks for detonation + expansion to ring 2 - game.tick().unwrap(); - game.tick().unwrap(); - game.tick().unwrap(); // detonates - game.tick().unwrap(); // ring 1 - let r3 = game.tick().unwrap(); // ring 2 → Charlie killed + game.tick().await.unwrap(); + game.tick().await.unwrap(); + game.tick().await.unwrap(); // detonates + game.tick().await.unwrap(); // ring 1 + let r3 = game.tick().await.unwrap(); // ring 2 → Charlie killed - assert!(!p3.is_alive().unwrap(), "Charlie should be dead"); + assert!(!p3.is_alive().await.unwrap(), "Charlie should be dead"); assert!(r3.game_over, "Game should be over with only Alice alive"); assert_eq!(r3.winner.as_deref(), Some("Alice")); // Verify final board - let board = game.board().unwrap(); + let board = game.board().await.unwrap(); let alive_count = board.data.players.iter().filter(|p| p.alive).count(); assert_eq!(alive_count, 1); let winner_state = board.data.players.iter().find(|p| p.alive).unwrap(); diff --git a/tests/bomberman/sim.rs b/tests/bomberman/sim.rs index 52de8bbb0..3b76c5564 100644 --- a/tests/bomberman/sim.rs +++ b/tests/bomberman/sim.rs @@ -1,6 +1,7 @@ use sourced_rust::{ - Commit, Get, ReadModelWorkspaceExt, ReadModelWritePlanStore, RelationalReadModelQueryStore, - RepositoryError, RowKey, RowValue, TransactionalCommit, Versioned, + AsyncGetStream, AsyncReadModelWorkspaceExt, AsyncReadModelWritePlanStore, + AsyncRelationalReadModelQueryStore, AsyncTransactionalCommit, RepositoryError, RowKey, + RowValue, Versioned, }; use crate::domain::player::Player; @@ -17,10 +18,13 @@ pub struct Game<'a, R> { impl<'a, R> Game<'a, R> where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - pub fn new(repo: &'a R, game_id: &str, ascii: &str) -> Result { - handlers::create_game(repo, game_id, ascii)?; + pub async fn new(repo: &'a R, game_id: &str, ascii: &str) -> Result { + handlers::create_game(repo, game_id, ascii).await?; Ok(Self { repo, game_id: game_id.to_string(), @@ -35,18 +39,19 @@ where } } - pub fn tick(&self) -> Result { - handlers::tick(self.repo, &self.game_id) + pub async fn tick(&self) -> Result { + handlers::tick(self.repo, &self.game_id).await } - pub fn board(&self) -> Result, GameError> { + pub async fn board(&self) -> Result, GameError> { self.repo - .workspace() - .load::(RowKey::new([( + .workspace_async() + .load_async::(RowKey::new([( "game_id", RowValue::String(self.game_id.clone()), )])) .one() + .await .map_err(|e| GameError::Repository(RepositoryError::Model(e.to_string())))? .ok_or(GameError::GameNotFound) } @@ -60,9 +65,12 @@ pub struct PlayerSim<'a, R> { impl<'a, R> PlayerSim<'a, R> where - R: Commit + TransactionalCommit + Get + ReadModelWritePlanStore + RelationalReadModelQueryStore, + R: AsyncGetStream + + AsyncTransactionalCommit + + AsyncReadModelWritePlanStore + + AsyncRelationalReadModelQueryStore, { - pub fn join(&self, spawn_index: usize) -> Result<(), GameError> { + pub async fn join(&self, spawn_index: usize) -> Result<(), GameError> { handlers::join_game( self.game.repo, &self.id, @@ -70,22 +78,23 @@ where &self.game.game_id, spawn_index, ) + .await } - pub fn move_dir(&self, dir: Direction) -> Result<(), GameError> { - handlers::move_player(self.game.repo, &self.id, dir, &self.game.game_id) + pub async fn move_dir(&self, dir: Direction) -> Result<(), GameError> { + handlers::move_player(self.game.repo, &self.id, dir, &self.game.game_id).await } - pub fn place_bomb(&self) -> Result<(), GameError> { - handlers::place_bomb(self.game.repo, &self.id, &self.game.game_id) + pub async fn place_bomb(&self) -> Result<(), GameError> { + handlers::place_bomb(self.game.repo, &self.id, &self.game.game_id).await } - pub fn is_alive(&self) -> Result { - let player = self.player()?; + pub async fn is_alive(&self) -> Result { + let player = self.player().await?; Ok(player.alive) } - pub fn player(&self) -> Result { - handlers::get_player(self.game.repo, &self.id) + pub async fn player(&self) -> Result { + handlers::get_player(self.game.repo, &self.id).await } } diff --git a/tests/read_model_commit_bridge/main.rs b/tests/read_model_commit_bridge/main.rs index 6c9dd4ead..5637ca819 100644 --- a/tests/read_model_commit_bridge/main.rs +++ b/tests/read_model_commit_bridge/main.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use sourced_rust::{ - sourced, Entity, HashMapRepository, ReadModel, ReadModelWorkspaceExt, - ReadModelWritePlanBuilder, RowKey, RowValue, SyncReadModelWritePlanCommitExt, + sourced, AsyncReadModelWorkspaceExt, AsyncReadModelWritePlanCommitExt, Entity, + HashMapRepository, ReadModel, ReadModelWritePlanBuilder, RowKey, RowValue, }; #[derive(Default)] @@ -27,8 +27,8 @@ struct BridgeView { value: i32, } -#[test] -fn repo_first_read_models_session_commit_form_is_available() { +#[tokio::test] +async fn repo_first_read_models_session_commit_form_is_available() { let repo = HashMapRepository::new(); let view = BridgeView { id: "view-1".into(), @@ -39,14 +39,16 @@ fn repo_first_read_models_session_commit_form_is_available() { let mut aggregate = TestAggregate::default(); aggregate.touch().unwrap(); - repo.read_models_sync(session) - .commit_sync(&mut aggregate) + repo.read_models(session) + .commit(&mut aggregate) + .await .unwrap(); let loaded = repo - .workspace() - .load::(RowKey::new([("id", RowValue::String("view-1".into()))])) + .workspace_async() + .load_async::(RowKey::new([("id", RowValue::String("view-1".into()))])) .one() + .await .unwrap() .unwrap(); assert_eq!(loaded.data, view); diff --git a/tests/read_model_relationship_includes/main.rs b/tests/read_model_relationship_includes/main.rs index 38f46e069..7faa531d1 100644 --- a/tests/read_model_relationship_includes/main.rs +++ b/tests/read_model_relationship_includes/main.rs @@ -1,11 +1,32 @@ +use std::future::Future; + use serde::{Deserialize, Serialize}; use sourced_rust::{ - AsyncReadModelWorkspaceExt, InMemoryReadModelStore, ReadModel, ReadModelAdapterCapabilities, - ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, - ReadModelQueryCapabilities, ReadModelWorkspaceExt, ReadModelWritePlan, ReadModelWritePlanStore, - RelationalReadModelQueryStore, RowKey, RowValue, + AsyncReadModelWorkspaceExt, AsyncReadModelWritePlanStore, AsyncRelationalReadModelQueryStore, + InMemoryReadModelStore, ReadModel, ReadModelAdapterCapabilities, ReadModelCommitOutcome, + ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, + ReadModelWritePlan, RowKey, RowValue, }; +fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] #[table("players")] struct Player { @@ -77,30 +98,30 @@ impl NoIncludeStore { } } -impl ReadModelWritePlanStore for NoIncludeStore { - fn read_model_capabilities(&self) -> ReadModelAdapterCapabilities { - self.inner.read_model_capabilities() +impl AsyncReadModelWritePlanStore for NoIncludeStore { + fn read_model_capabilities_async(&self) -> ReadModelAdapterCapabilities { + self.inner.read_model_capabilities_async() } - fn commit_write_plan( + fn commit_write_plan_async( &self, plan: ReadModelWritePlan, - ) -> Result { - self.inner.commit_write_plan(plan) + ) -> impl Future> + Send + '_ { + self.inner.commit_write_plan_async(plan) } } -impl RelationalReadModelQueryStore for NoIncludeStore { - fn read_model_query_capabilities(&self) -> ReadModelQueryCapabilities { +impl AsyncRelationalReadModelQueryStore for NoIncludeStore { + fn read_model_query_capabilities_async(&self) -> ReadModelQueryCapabilities { ReadModelQueryCapabilities::default() } - fn load_graph( + async fn load_graph_async( &self, request: ReadModelLoadRequest, ) -> Result { - request.validate_for_query_capabilities(&self.read_model_query_capabilities())?; - self.inner.load_graph(request) + request.validate_for_query_capabilities(&self.read_model_query_capabilities_async())?; + self.inner.load_graph_async(request).await } } @@ -144,19 +165,21 @@ fn store_with_player_and_weapons( for weapon in weapons { session.upsert(&weapon).unwrap(); } - session.commit(&store).unwrap(); + block_on(session.commit_async(&store)).unwrap(); store } #[test] fn friendly_session_loads_one_root_by_primary_key_without_includes() { let store = store_with_player_and_weapons([]); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let loaded = read_models - .load::(player_key("player-1")) - .one() - .unwrap(); + let loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .one(), + ) + .unwrap(); assert_eq!(loaded.unwrap().data.display_name, "Ada"); } @@ -164,14 +187,16 @@ fn friendly_session_loads_one_root_by_primary_key_without_includes() { #[test] fn friendly_session_hydrates_has_many_include() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let loaded = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap(); + let loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap(); assert_eq!(loaded.data.weapons[0].weapon_id, "sword"); } @@ -179,14 +204,16 @@ fn friendly_session_hydrates_has_many_include() { #[test] fn friendly_session_hydrates_belongs_to_include() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let loaded = read_models - .load::(weapon_key("player-1", "sword")) - .include("player") - .one() - .unwrap() - .unwrap(); + let loaded = block_on( + read_models + .load_async::(weapon_key("player-1", "sword")) + .include("player") + .one(), + ) + .unwrap() + .unwrap(); assert_eq!(loaded.data.player.unwrap().display_name, "Ada"); } @@ -194,22 +221,22 @@ fn friendly_session_hydrates_belongs_to_include() { #[test] fn sync_persists_loaded_scalar_field_without_manual_patch() { let store = store_with_player_and_weapons([]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(player_key("player-1")) - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .one(), + ) + .unwrap() + .unwrap() + .data; loaded.display_name = "Ada Lovelace".into(); read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let reloaded = check - .load::(player_key("player-1")) - .one() + let mut check = store.workspace_async(); + let reloaded = block_on(check.load_async::(player_key("player-1")).one()) .unwrap() .unwrap(); assert_eq!(reloaded.data.display_name, "Ada Lovelace"); @@ -218,24 +245,24 @@ fn sync_persists_loaded_scalar_field_without_manual_patch() { #[test] fn sync_refreshes_loaded_root_baseline_between_calls() { let store = store_with_player_and_weapons([]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(player_key("player-1")) - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .one(), + ) + .unwrap() + .unwrap() + .data; loaded.display_name = "Ada Lovelace".into(); read_models.sync(loaded.clone()).unwrap(); loaded.display_name = "Countess Lovelace".into(); read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let reloaded = check - .load::(player_key("player-1")) - .one() + let mut check = store.workspace_async(); + let reloaded = block_on(check.load_async::(player_key("player-1")).one()) .unwrap() .unwrap(); assert_eq!(reloaded.data.display_name, "Countess Lovelace"); @@ -244,28 +271,32 @@ fn sync_refreshes_loaded_root_baseline_between_calls() { #[test] fn sync_persists_added_and_modified_related_rows() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap() + .data; loaded.weapons[0].acquired_at = "2026-05-24".into(); loaded.weapons.push(weapon("", "shield", "2026-05-25")); read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let mut reloaded = check - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap() - .data; + let mut check = store.workspace_async(); + let mut reloaded = block_on( + check + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap() + .data; reloaded .weapons .sort_by(|left, right| left.weapon_id.cmp(&right.weapon_id)); @@ -277,29 +308,33 @@ fn sync_persists_added_and_modified_related_rows() { #[test] fn sync_refreshes_loaded_include_baseline_between_calls() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap() + .data; loaded.weapons[0].acquired_at = "2026-05-24".into(); read_models.sync(loaded.clone()).unwrap(); loaded.weapons[0].acquired_at = "2026-05-25".into(); read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let reloaded = check - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap() - .data; + let mut check = store.workspace_async(); + let reloaded = block_on( + check + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap() + .data; assert_eq!(reloaded.weapons[0].acquired_at, "2026-05-25"); } @@ -309,26 +344,30 @@ fn sync_deletes_removed_related_rows() { weapon("player-1", "shield", "2026-05-24"), weapon("player-1", "sword", "2026-05-23"), ]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap() + .data; loaded.weapons.retain(|weapon| weapon.weapon_id == "sword"); read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let reloaded = check - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap() - .unwrap(); + let mut check = store.workspace_async(); + let reloaded = block_on( + check + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap() + .unwrap(); assert_eq!(reloaded.data.weapons.len(), 1); assert_eq!(reloaded.data.weapons[0].weapon_id, "sword"); } @@ -336,35 +375,39 @@ fn sync_deletes_removed_related_rows() { #[test] fn sync_clearing_belongs_to_does_not_delete_target() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); - let mut loaded = read_models - .load::(weapon_key("player-1", "sword")) - .include("player") - .one() - .unwrap() - .unwrap() - .data; + let mut read_models = store.workspace_async(); + let mut loaded = block_on( + read_models + .load_async::(weapon_key("player-1", "sword")) + .include("player") + .one(), + ) + .unwrap() + .unwrap() + .data; assert!(loaded.player.is_some()); loaded.player = None; read_models.sync(loaded).unwrap(); - read_models.commit().unwrap(); + block_on(read_models.commit_async()).unwrap(); - let mut check = store.workspace(); - let player = check.load::(player_key("player-1")).one().unwrap(); + let mut check = store.workspace_async(); + let player = block_on(check.load_async::(player_key("player-1")).one()).unwrap(); assert_eq!(player.unwrap().data.display_name, "Ada"); } #[test] fn missing_root_returns_none_without_include_loading() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let loaded = read_models - .load::(player_key("missing")) - .include("weapons") - .one() - .unwrap(); + let loaded = block_on( + read_models + .load_async::(player_key("missing")) + .include("weapons") + .one(), + ) + .unwrap(); assert!(loaded.is_none()); } @@ -375,14 +418,16 @@ fn unregistered_relationship_target_fails_before_loading() { store.register_schema::().unwrap(); let mut session = sourced_rust::ReadModelWritePlanBuilder::new(); session.upsert(&player("player-1", "Ada")).unwrap(); - session.commit(&store).unwrap(); - let mut read_models = store.workspace(); + block_on(session.commit_async(&store)).unwrap(); + let mut read_models = store.workspace_async(); - let err = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap_err(); + let err = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("unregistered model `PlayerWeapon`")) @@ -394,14 +439,16 @@ fn unregistered_root_schema_can_load_primary_key_without_includes() { let store = InMemoryReadModelStore::new(); let mut session = sourced_rust::ReadModelWritePlanBuilder::new(); session.upsert(&player("player-1", "Ada")).unwrap(); - session.commit(&store).unwrap(); - let mut read_models = store.workspace(); + block_on(session.commit_async(&store)).unwrap(); + let mut read_models = store.workspace_async(); - let loaded = read_models - .load::(player_key("player-1")) - .one() - .unwrap() - .unwrap(); + let loaded = block_on( + read_models + .load_async::(player_key("player-1")) + .one(), + ) + .unwrap() + .unwrap(); assert_eq!(loaded.data.display_name, "Ada"); } @@ -410,13 +457,15 @@ fn unregistered_root_schema_can_load_primary_key_without_includes() { fn adapter_without_include_capability_rejects_includes() { let inner = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); let store = NoIncludeStore::new(inner); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let err = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap_err(); + let err = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("relationship includes")) @@ -426,13 +475,15 @@ fn adapter_without_include_capability_rejects_includes() { #[test] fn nested_query_style_include_paths_are_not_a_public_query_dsl() { let store = store_with_player_and_weapons([weapon("player-1", "sword", "2026-05-23")]); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let err = read_models - .load::(player_key("player-1")) - .include("weapons.owner") - .one() - .unwrap_err(); + let err = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons.owner") + .one(), + ) + .unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("has no relationship")) @@ -443,13 +494,15 @@ fn nested_query_style_include_paths_are_not_a_public_query_dsl() { fn many_to_many_include_fails_until_join_metadata_is_rich_enough() { let store = InMemoryReadModelStore::new(); store.register_schema::().unwrap(); - let mut read_models = store.workspace(); + let mut read_models = store.workspace_async(); - let err = read_models - .load::(player_key("player-1")) - .include("weapons") - .one() - .unwrap_err(); + let err = block_on( + read_models + .load_async::(player_key("player-1")) + .include("weapons") + .one(), + ) + .unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("many-to-many relationship")) @@ -475,14 +528,19 @@ fn belongs_to_include_rejects_composite_target_primary_key() { label: "Sword".into(), }) .unwrap(); - session.commit(&store).unwrap(); - let mut read_models = store.workspace(); + block_on(session.commit_async(&store)).unwrap(); + let mut read_models = store.workspace_async(); - let err = read_models - .load::(RowKey::new([("ref_id", RowValue::String("ref-1".into()))])) - .include("label") - .one() - .unwrap_err(); + let err = block_on( + read_models + .load_async::(RowKey::new([( + "ref_id", + RowValue::String("ref-1".into()), + )])) + .include("label") + .one(), + ) + .unwrap_err(); assert!( matches!(err, ReadModelError::Metadata(message) if message.contains("CompositeWeaponLabel") @@ -495,8 +553,7 @@ fn belongs_to_include_rejects_composite_target_primary_key() { // // `InMemoryReadModelStore` implements the async store traits, so the same // workspace ergonomic is available over `workspace_async()` / -// `load_async()` / `commit_async()`. These mirror the sync `include` + -// `sync`-roundtrip tests above to prove the async path is at parity. +// `load_async()` / `commit_async()`. #[tokio::test] async fn async_session_hydrates_has_many_include() { diff --git a/tests/sourced_upcasting/main.rs b/tests/sourced_upcasting/main.rs index bce0842b2..07e21e81c 100644 --- a/tests/sourced_upcasting/main.rs +++ b/tests/sourced_upcasting/main.rs @@ -1,7 +1,10 @@ mod aggregate; use aggregate::{TodoV1, TodoV1Event, TodoV2, TodoV2Event, TodoV3}; -use sourced_rust::{hydrate, Aggregate, AggregateBuilder, Commit, Entity, HashMapRepository}; +use sourced_rust::{ + hydrate, Aggregate, AsyncAggregateBuilder, AsyncCommitBatch, AsyncStreamWrite, + AsyncTransactionalCommit, Entity, HashMapRepository, StreamIdentity, +}; #[test] fn v1_has_no_upcasters() { @@ -92,16 +95,24 @@ fn hydrate_v3_native_no_upcasting() { assert_eq!(loaded.due_date, "2025-12-31"); } -#[test] -fn repo_roundtrip_v1_to_v2() { +#[tokio::test] +async fn repo_roundtrip_v1_to_v2() { let repo = HashMapRepository::new(); let mut v1 = TodoV1::default(); v1.initialize("t1".into(), "frank".into(), "Shop".into()) .unwrap(); - repo.commit(&mut v1.entity).unwrap(); + // Store the v1 events under the v2 aggregate type so the v2 repository (which + // upcasts on load) reads the same stream. + let identity = StreamIdentity::new(TodoV2::aggregate_type(), "t1").unwrap(); + repo.commit_batch_async(AsyncCommitBatch::new(vec![AsyncStreamWrite::new( + identity, + &mut v1.entity, + )])) + .await + .unwrap(); - let v2_repo = repo.aggregate::(); - let loaded = v2_repo.get("t1").unwrap().unwrap(); + let v2_repo = repo.async_aggregate::(); + let loaded = v2_repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.user_id, "frank"); assert_eq!(loaded.priority, 0); } diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs index fa0264891..5a0a52dce 100644 --- a/tests/transport_conformance/mod.rs +++ b/tests/transport_conformance/mod.rs @@ -22,8 +22,8 @@ use sourced_rust::microsvc::transport::{ }; use sourced_rust::microsvc::{Context, HandlerError, Message, MessageKind, Service}; use sourced_rust::{ - CommitBatch, HashMapOutboxStore, HashMapRepository, OutboxMessage, OutboxMessageStatus, - TransactionalCommit, + AsyncCommitBatch, AsyncTransactionalCommit, HashMapOutboxStore, HashMapRepository, + OutboxMessage, OutboxMessageStatus, }; /// One observable transport effect, recorded in order. @@ -348,11 +348,11 @@ pub async fn source_propagates_settle_errors() { // Publisher / outbox dispatcher contract // ============================================================================= -fn store_outbox(repo: &HashMapRepository, id: &str) -> String { +async fn store_outbox(repo: &HashMapRepository, id: &str) -> String { let message = OutboxMessage::create(id, "OrderCreated", b"\x01".to_vec()).unwrap(); - let mut batch = CommitBatch::empty(); + let mut batch = AsyncCommitBatch::empty(); batch.outbox_messages.push(message); - repo.commit_batch(batch).unwrap(); + repo.commit_batch_async(batch).await.unwrap(); id.to_string() } @@ -391,7 +391,7 @@ fn dispatcher( pub async fn dispatcher_completes_only_after_publish_success() { let repo = HashMapRepository::new(); - let id = store_outbox(&repo, "evt-1"); + let id = store_outbox(&repo, "evt-1").await; let dispatcher = dispatcher(&repo, PublishMode::Succeed, 3); let outcome = dispatcher @@ -411,7 +411,7 @@ pub async fn dispatcher_completes_only_after_publish_success() { pub async fn dispatcher_unknown_outcome_stays_retryable() { let repo = HashMapRepository::new(); - let id = store_outbox(&repo, "evt-1"); + let id = store_outbox(&repo, "evt-1").await; let dispatcher = dispatcher(&repo, PublishMode::FailUnknown, 3); let outcome = dispatcher @@ -429,8 +429,8 @@ pub async fn dispatcher_unknown_outcome_stays_retryable() { pub async fn dispatcher_claims_explicit_ids_before_publish() { let repo = HashMapRepository::new(); - let wanted = store_outbox(&repo, "evt-1"); - let other = store_outbox(&repo, "evt-2"); + let wanted = store_outbox(&repo, "evt-1").await; + let other = store_outbox(&repo, "evt-2").await; let dispatcher = dispatcher(&repo, PublishMode::Succeed, 3); let outcome = dispatcher From e2da936bf2057f69e9ad251ce570a40e293ddd3f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 20:50:48 -0500 Subject: [PATCH 38/39] test: gate matrix table_schema_registry helper to postgres/sqlite It is only used by the postgres/sqlite-gated matrix cells, so it (and its `TableSchemaRegistry` import) tripped a dead-code warning on the default build. Gate both with cfg(any(feature = "postgres", feature = "sqlite")) to match the call sites. Default clippy is now fully clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/distributed_read_model/read_models/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/distributed_read_model/read_models/mod.rs b/tests/distributed_read_model/read_models/mod.rs index a5817e5c2..1dddd1e45 100644 --- a/tests/distributed_read_model/read_models/mod.rs +++ b/tests/distributed_read_model/read_models/mod.rs @@ -10,7 +10,9 @@ pub use checkout_step_view::CheckoutStepView; pub use checkout_view::CheckoutView; pub use seat_view::SeatView; -use sourced_rust::{InMemoryReadModelStore, ReadModelError, RowKey, RowValue, TableSchemaRegistry}; +#[cfg(any(feature = "postgres", feature = "sqlite"))] +use sourced_rust::TableSchemaRegistry; +use sourced_rust::{InMemoryReadModelStore, ReadModelError, RowKey, RowValue}; pub fn register_schemas(store: &InMemoryReadModelStore) -> Result<(), ReadModelError> { store.register_schema::()?; @@ -19,6 +21,7 @@ pub fn register_schemas(store: &InMemoryReadModelStore) -> Result<(), ReadModelE Ok(()) } +#[cfg(any(feature = "postgres", feature = "sqlite"))] pub fn table_schema_registry() -> Result { let mut registry = TableSchemaRegistry::new(); registry.register::()?; From 47466ea437019d51941db5bbbec425f16617d7c9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 29 May 2026 21:42:18 -0500 Subject: [PATCH 39/39] test: address CodeRabbit review (block_on, handler panic, weak assertion) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/ unit tests (outbox/commit, snapshot/in_memory, snapshot/repository, read_model/in_memory, commit_builder, outbox_worker/store, hashmap_repo): replace the custom busy-poll `block_on` (no-op waker, ignores Poll::Pending — would spin on any yielding future) with `#[tokio::test]`. Transport modules keep their intentionally runtime-free block_on. - board projection handler: `event_version` returns Result<_, HandlerError> instead of panicking on a malformed message id; the handler propagates with `?`. Its unit test now asserts the error path. - tests/todos: the bulk-commit roundtrip now asserts the commit succeeds and that exactly 3 todos are present (was: ignored result + an `if !empty` that masked failures). The concurrency-race commits (deliberately may lose the lock) keep their `let _ =`. 490 tests pass; clippy --all-targets clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/commit_builder/mod.rs | 243 ++++++++---------- src/hashmap_repo/repository.rs | 88 +++---- src/outbox/commit.rs | 31 +-- src/outbox_worker/store.rs | 95 +++---- src/read_model/in_memory.rs | 111 ++++---- src/snapshot/in_memory.rs | 128 ++++----- src/snapshot/repository.rs | 25 +- .../projections_service/handlers/board.rs | 29 ++- tests/todos/main.rs | 14 +- 9 files changed, 338 insertions(+), 426 deletions(-) diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index f5a42667a..66dd55db7 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -322,25 +322,6 @@ mod tests { use serde::{Deserialize, Serialize}; use std::sync::Mutex; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - type OutboxSourceRecord = (String, Option, Option, Option); #[derive(Default)] @@ -452,19 +433,18 @@ mod tests { .lock_key() } - fn loaded_view(repo: &HashMapRepository, id: &str) -> Option { - block_on( - repo.model_store() - .workspace_async() - .load_async::(view_key(id)) - .one(), - ) - .unwrap() - .map(|versioned| versioned.data) + async fn loaded_view(repo: &HashMapRepository, id: &str) -> Option { + repo.model_store() + .workspace_async() + .load_async::(view_key(id)) + .one() + .await + .unwrap() + .map(|versioned| versioned.data) } - #[test] - fn commit_read_models_and_aggregate() { + #[tokio::test] + async fn commit_read_models_and_aggregate() { let repo = HashMapRepository::new(); let view = RelationalView { @@ -475,19 +455,18 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .commit(&mut agg), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .commit(&mut agg) + .await + .unwrap(); - let loaded = loaded_view(&repo, "1").unwrap(); + let loaded = loaded_view(&repo, "1").await.unwrap(); assert_eq!(loaded.counter, 42); assert_eq!(agg.entity().committed_version(), 1); } - #[test] - fn commit_multiple_read_models() { + #[tokio::test] + async fn commit_multiple_read_models() { let repo = HashMapRepository::new(); let view1 = RelationalView { @@ -505,17 +484,17 @@ mod tests { let mut read_models = crate::read_model::ReadModelWritePlanBuilder::new(); read_models.upsert(&view1).unwrap().upsert(&view2).unwrap(); - block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models).commit(&mut agg), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models) + .commit(&mut agg) + .await + .unwrap(); - assert_eq!(loaded_view(&repo, "1").unwrap().counter, 10); - assert_eq!(loaded_view(&repo, "2").unwrap().counter, 20); + assert_eq!(loaded_view(&repo, "1").await.unwrap().counter, 10); + assert_eq!(loaded_view(&repo, "2").await.unwrap().counter, 20); } - #[test] - fn commit_read_models_with_outbox() { + #[tokio::test] + async fn commit_read_models_with_outbox() { let repo = HashMapRepository::new(); let view = RelationalView { @@ -528,18 +507,17 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .outbox(outbox) - .commit(&mut agg), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .commit(&mut agg) + .await + .unwrap(); - assert_eq!(loaded_view(&repo, "1").unwrap().counter, 42); + assert_eq!(loaded_view(&repo, "1").await.unwrap().counter, 42); } - #[test] - fn commit_outbox_then_read_models() { + #[tokio::test] + async fn commit_outbox_then_read_models() { let repo = HashMapRepository::new(); let view = RelationalView { @@ -552,18 +530,17 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - block_on( - AsyncCommitBuilderExt::outbox(&repo, outbox) - .read_models(read_models(&view)) - .commit(&mut agg), - ) - .unwrap(); + AsyncCommitBuilderExt::outbox(&repo, outbox) + .read_models(read_models(&view)) + .commit(&mut agg) + .await + .unwrap(); - assert_eq!(loaded_view(&repo, "1").unwrap().counter, 99); + assert_eq!(loaded_view(&repo, "1").await.unwrap().counter, 99); } - #[test] - fn commit_all_without_aggregate() { + #[tokio::test] + async fn commit_all_without_aggregate() { let repo = HashMapRepository::new(); let view1 = RelationalView { @@ -578,21 +555,23 @@ mod tests { let mut read_models = crate::read_model::ReadModelWritePlanBuilder::new(); read_models.upsert(&view1).unwrap().upsert(&view2).unwrap(); - block_on(AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models).commit_all()) + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models) + .commit_all() + .await .unwrap(); assert_eq!( - loaded_view(&repo, "standalone-1").unwrap().id, + loaded_view(&repo, "standalone-1").await.unwrap().id, "standalone-1" ); assert_eq!( - loaded_view(&repo, "standalone-2").unwrap().id, + loaded_view(&repo, "standalone-2").await.unwrap().id, "standalone-2" ); } - #[test] - fn commit_many_multiple_aggregates() { + #[tokio::test] + async fn commit_many_multiple_aggregates() { let repo = HashMapRepository::new(); let view = RelationalView { @@ -608,32 +587,33 @@ mod tests { agg2.touch().unwrap(); agg2.entity.set_id("agg-2"); - block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .commit_many(&mut [&mut agg1, &mut agg2]), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .commit_many(&mut [&mut agg1, &mut agg2]) + .await + .unwrap(); - assert_eq!(loaded_view(&repo, "multi").unwrap().counter, 77); + assert_eq!(loaded_view(&repo, "multi").await.unwrap().counter, 77); let agg_type = TestAggregate::aggregate_type(); - let e1 = block_on(crate::AsyncGetStream::get_stream( + let e1 = crate::AsyncGetStream::get_stream( &repo, &StreamIdentity::new(agg_type, "agg-1").unwrap(), - )) + ) + .await .unwrap(); assert!(e1.is_some()); - let e2 = block_on(crate::AsyncGetStream::get_stream( + let e2 = crate::AsyncGetStream::get_stream( &repo, &StreamIdentity::new(agg_type, "agg-2").unwrap(), - )) + ) + .await .unwrap(); assert!(e2.is_some()); } - #[test] - fn staged_builder_ordering_is_semantic_for_outbox_session_and_aggregate() { - fn record(order: u8) -> (Vec<(String, String)>, Vec) { + #[tokio::test] + async fn staged_builder_ordering_is_semantic_for_outbox_session_and_aggregate() { + async fn record(order: u8) -> (Vec<(String, String)>, Vec) { let repo = RecordingAsyncBatchRepo::default(); let view = RelationalView { id: "ordered".into(), @@ -644,27 +624,25 @@ mod tests { agg.touch().unwrap(); match order { - 0 => block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .outbox(outbox) - .aggregate(&mut agg) - .commit(), - ) - .unwrap(), - 1 => block_on( - repo.outbox(outbox) - .read_models(read_models(&view)) - .aggregate(&mut agg) - .commit(), - ) - .unwrap(), - _ => block_on( - AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) - .read_models(read_models(&view)) - .outbox(outbox) - .commit(), - ) - .unwrap(), + 0 => AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .aggregate(&mut agg) + .commit() + .await + .unwrap(), + 1 => repo + .outbox(outbox) + .read_models(read_models(&view)) + .aggregate(&mut agg) + .commit() + .await + .unwrap(), + _ => AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) + .read_models(read_models(&view)) + .outbox(outbox) + .commit() + .await + .unwrap(), } let stream_ids = repo.stream_ids.lock().unwrap().clone(); @@ -672,24 +650,23 @@ mod tests { (stream_ids, read_model_keys) } - let baseline = record(0); - assert_eq!(record(1), baseline); - assert_eq!(record(2), baseline); + let baseline = record(0).await; + assert_eq!(record(1).await, baseline); + assert_eq!(record(2).await, baseline); } - #[test] - fn staged_commit_sets_outbox_source_from_single_aggregate() { + #[tokio::test] + async fn staged_commit_sets_outbox_source_from_single_aggregate() { let repo = RecordingAsyncBatchRepo::default(); let mut agg = TestAggregate::default(); agg.touch().unwrap(); let outbox = OutboxMessage::create("sourced-msg", "TestEvent", b"{}".to_vec()).unwrap(); - block_on( - AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) - .outbox(outbox) - .commit(), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::aggregate(&repo, &mut agg) + .outbox(outbox) + .commit() + .await + .unwrap(); assert_eq!( repo.outbox_sources.lock().unwrap().as_slice(), @@ -702,8 +679,8 @@ mod tests { ); } - #[test] - fn staged_builder_supports_multiple_aggregates() { + #[tokio::test] + async fn staged_builder_supports_multiple_aggregates() { let repo = RecordingAsyncBatchRepo::default(); let view = RelationalView { id: "staged-multi".into(), @@ -716,13 +693,12 @@ mod tests { agg2.touch().unwrap(); agg2.entity.set_id("agg-2"); - block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .aggregate(&mut agg1) - .aggregate(&mut agg2) - .commit(), - ) - .unwrap(); + AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .aggregate(&mut agg1) + .aggregate(&mut agg2) + .commit() + .await + .unwrap(); assert_eq!( repo.read_model_keys.lock().unwrap().as_slice(), @@ -743,8 +719,8 @@ mod tests { ); } - #[test] - fn commit_builder_failure_does_not_mark_aggregate_committed() { + #[tokio::test] + async fn commit_builder_failure_does_not_mark_aggregate_committed() { let repo = RecordingAsyncBatchRepo { fail: true, ..Default::default() @@ -758,12 +734,11 @@ mod tests { let mut agg = TestAggregate::default(); agg.touch().unwrap(); - let err = block_on( - AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) - .outbox(outbox) - .commit(&mut agg), - ) - .unwrap_err(); + let err = AsyncReadModelWritePlanCommitExt::read_models(&repo, read_models(&view)) + .outbox(outbox) + .commit(&mut agg) + .await + .unwrap_err(); assert_eq!( err, @@ -789,11 +764,11 @@ mod tests { .any(|id| id == "msg-rollback")); } - #[test] - fn commit_builder_empty_batch_succeeds() { + #[tokio::test] + async fn commit_builder_empty_batch_succeeds() { let repo = RecordingAsyncBatchRepo::default(); - block_on(AsyncCommitBuilder::new(&repo).commit_all()).unwrap(); + AsyncCommitBuilder::new(&repo).commit_all().await.unwrap(); assert!(repo.stream_ids.lock().unwrap().is_empty()); assert!(repo.read_model_keys.lock().unwrap().is_empty()); diff --git a/src/hashmap_repo/repository.rs b/src/hashmap_repo/repository.rs index cf5275f5e..9aa6d43d1 100644 --- a/src/hashmap_repo/repository.rs +++ b/src/hashmap_repo/repository.rs @@ -435,37 +435,20 @@ impl AsyncSnapshotStore for HashMapRepository { mod tests { use super::*; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - fn identity(id: &str) -> StreamIdentity { StreamIdentity::new("test.aggregate", id).unwrap() } - fn commit_one(repo: &HashMapRepository, entity: &mut Entity) -> Result<(), RepositoryError> { + async fn commit_one( + repo: &HashMapRepository, + entity: &mut Entity, + ) -> Result<(), RepositoryError> { let id = entity.id().to_string(); - block_on( - repo.commit_batch_async(AsyncCommitBatch::new(vec![AsyncStreamWrite::new( - identity(&id), - entity, - )])), - ) + repo.commit_batch_async(AsyncCommitBatch::new(vec![AsyncStreamWrite::new( + identity(&id), + entity, + )])) + .await } #[test] @@ -474,23 +457,23 @@ mod tests { assert!(repo.event_store.read().unwrap().is_empty()); } - #[test] - fn single_entity_commit() { + #[tokio::test] + async fn single_entity_commit() { let repo = HashMapRepository::new(); let id = "test_id"; let mut entity = Entity::with_id(id); entity.digest("test_event", &("arg1", "arg2")).unwrap(); - commit_one(&repo, &mut entity).unwrap(); + commit_one(&repo, &mut entity).await.unwrap(); - let fetched_entity = block_on(repo.get_stream(&identity(id))).unwrap().unwrap(); + let fetched_entity = repo.get_stream(&identity(id)).await.unwrap().unwrap(); assert_eq!(fetched_entity.id(), id); assert_eq!(fetched_entity.events(), entity.events()); } - #[test] - fn multiple_entity_commit() { + #[tokio::test] + async fn multiple_entity_commit() { let repo = HashMapRepository::new(); let mut entity1 = Entity::with_id("id_1"); @@ -499,19 +482,22 @@ mod tests { let mut entity2 = Entity::with_id("id_2"); entity2.digest("event2", &"arg2").unwrap(); - block_on(repo.commit_batch_async(AsyncCommitBatch::new(vec![ + repo.commit_batch_async(AsyncCommitBatch::new(vec![ AsyncStreamWrite::new(identity("id_1"), &mut entity1), AsyncStreamWrite::new(identity("id_2"), &mut entity2), - ]))) + ])) + .await .unwrap(); - let all_entities: Vec = - block_on(repo.get_streams(&[identity("id_1"), identity("id_2")])).unwrap(); + let all_entities: Vec = repo + .get_streams(&[identity("id_1"), identity("id_2")]) + .await + .unwrap(); assert_eq!(all_entities.len(), 2); } - #[test] - fn duplicate_stream_ids_rejected_before_write() { + #[tokio::test] + async fn duplicate_stream_ids_rejected_before_write() { let repo = HashMapRepository::new(); let mut entity1 = Entity::with_id("same-id"); @@ -520,11 +506,13 @@ mod tests { let mut entity2 = Entity::with_id("same-id"); entity2.digest("event2", &"arg2").unwrap(); - let err = block_on(repo.commit_batch_async(AsyncCommitBatch::new(vec![ - AsyncStreamWrite::new(identity("same-id"), &mut entity1), - AsyncStreamWrite::new(identity("same-id"), &mut entity2), - ]))) - .unwrap_err(); + let err = repo + .commit_batch_async(AsyncCommitBatch::new(vec![ + AsyncStreamWrite::new(identity("same-id"), &mut entity1), + AsyncStreamWrite::new(identity("same-id"), &mut entity2), + ])) + .await + .unwrap_err(); assert_eq!( err, RepositoryError::DuplicateStreamInBatch { @@ -532,7 +520,9 @@ mod tests { } ); - assert!(block_on(repo.get_stream(&identity("same-id"))) + assert!(repo + .get_stream(&identity("same-id")) + .await .unwrap() .is_none()); assert_eq!(entity1.committed_version(), 0); @@ -541,14 +531,14 @@ mod tests { assert_eq!(entity2.new_events().len(), 1); } - #[test] - fn inbox_receipts_record_dedupe_and_roll_back_atomically() { + #[tokio::test] + async fn inbox_receipts_record_dedupe_and_roll_back_atomically() { use crate::repository::InboxReceipt; let repo = HashMapRepository::new(); let mut batch = AsyncCommitBatch::empty(); batch.inbox_receipts.push(InboxReceipt::new("proj", "m1")); - block_on(repo.commit_batch_async(batch)).unwrap(); + repo.commit_batch_async(batch).await.unwrap(); assert!(repo.inbox_contains("proj", "m1")); assert!(!repo.inbox_contains("proj", "m2")); @@ -556,7 +546,7 @@ mod tests { let mut dup = AsyncCommitBatch::empty(); dup.inbox_receipts.push(InboxReceipt::new("proj", "m1")); dup.inbox_receipts.push(InboxReceipt::new("proj", "m2")); - let err = block_on(repo.commit_batch_async(dup)).unwrap_err(); + let err = repo.commit_batch_async(dup).await.unwrap_err(); assert!( matches!(err, RepositoryError::DuplicateInboxReceipt { ref message_id, .. } if message_id == "m1"), "got {err:?}" @@ -570,7 +560,7 @@ mod tests { let mut invalid = AsyncCommitBatch::empty(); invalid.inbox_receipts.push(InboxReceipt::new("", "m3")); assert!(matches!( - block_on(repo.commit_batch_async(invalid)).unwrap_err(), + repo.commit_batch_async(invalid).await.unwrap_err(), RepositoryError::InvalidInboxReceipt { .. } )); } diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index d00bb58f3..b24671d0b 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -46,25 +46,6 @@ mod tests { use crate::{sourced, AsyncAggregateBuilder, Entity, HashMapRepository, OutboxStore}; use std::sync::Mutex; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - #[derive(Default)] struct Dummy { entity: Entity, @@ -108,8 +89,8 @@ mod tests { } } - #[test] - fn outbox_helper_commits_both_entities() { + #[tokio::test] + async fn outbox_helper_commits_both_entities() { let repo = HashMapRepository::new().async_aggregate::(); let mut aggregate = Dummy::default(); @@ -117,15 +98,15 @@ mod tests { let event = OutboxMessage::create("msg-1", "DummyTouched", b"{}".to_vec()).unwrap(); - block_on(repo.outbox(event).commit(&mut aggregate)).unwrap(); + repo.outbox(event).commit(&mut aggregate).await.unwrap(); let pending = repo.repo().outbox_store().pending().unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].id(), "msg-1"); } - #[test] - fn outbox_helper_failure_leaves_entities_uncommitted() { + #[tokio::test] + async fn outbox_helper_failure_leaves_entities_uncommitted() { let repo = AsyncAggregateRepository::<_, Dummy>::new(FailingOutboxRepo::default()); let mut aggregate = Dummy::default(); @@ -133,7 +114,7 @@ mod tests { let event = OutboxMessage::create("msg-fail", "DummyTouched", b"{}".to_vec()).unwrap(); - let err = block_on(repo.outbox(event).commit(&mut aggregate)).unwrap_err(); + let err = repo.outbox(event).commit(&mut aggregate).await.unwrap_err(); assert_eq!(err, RepositoryError::Model("outbox write failed".into())); assert_eq!(aggregate.entity.committed_version(), 0); diff --git a/src/outbox_worker/store.rs b/src/outbox_worker/store.rs index 2ca74d899..7480fa363 100644 --- a/src/outbox_worker/store.rs +++ b/src/outbox_worker/store.rs @@ -530,34 +530,14 @@ impl AsyncOutboxStore for HashMapOutboxStore { mod tests { use super::*; use crate::{AsyncCommitBatch, AsyncTransactionalCommit, HashMapRepository}; - use std::future::Future; use std::sync::{Arc, Barrier}; use std::thread; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - - fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { + async fn store_message(repo: &HashMapRepository, message: OutboxMessage) -> String { let id = message.id().to_string(); let mut batch = AsyncCommitBatch::empty(); batch.outbox_messages.push(message); - block_on(repo.commit_batch_async(batch)).unwrap(); + repo.commit_batch_async(batch).await.unwrap(); id } @@ -570,14 +550,14 @@ mod tests { .clone() } - #[test] - fn claim_includes_expired_in_flight_messages() { + #[tokio::test] + async fn claim_includes_expired_in_flight_messages() { let repo = HashMapRepository::new(); let mut message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); message .claim_at("worker-1", Duration::from_secs(1), SystemTime::UNIX_EPOCH) .unwrap(); - let id = store_message(&repo, message); + let id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store @@ -598,14 +578,14 @@ mod tests { assert!(stored.is_in_flight()); } - #[test] - fn claim_skips_unexpired_in_flight_messages() { + #[tokio::test] + async fn claim_skips_unexpired_in_flight_messages() { let repo = HashMapRepository::new(); let mut message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); message .claim_for("worker-1", Duration::from_secs(60)) .unwrap(); - let id = store_message(&repo, message); + let id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store @@ -622,15 +602,15 @@ mod tests { assert_eq!(stored.attempts, 1); } - #[test] - fn claim_uses_created_at_before_message_id_order() { + #[tokio::test] + async fn claim_uses_created_at_before_message_id_order() { let repo = HashMapRepository::new(); let mut newer = OutboxMessage::create("msg-a", "Event", b"{}".to_vec()).unwrap(); newer.created_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10); let mut older = OutboxMessage::create("msg-z", "Event", b"{}".to_vec()).unwrap(); older.created_at = SystemTime::UNIX_EPOCH + Duration::from_secs(1); - store_message(&repo, newer); - store_message(&repo, older); + store_message(&repo, newer).await; + store_message(&repo, older).await; let claimed = repo .outbox_store() @@ -644,21 +624,24 @@ mod tests { assert_eq!(claimed[0].id(), "msg-z"); } - #[test] - fn claim_by_explicit_ids_claims_only_requested() { + #[tokio::test] + async fn claim_by_explicit_ids_claims_only_requested() { let repo = HashMapRepository::new(); store_message( &repo, OutboxMessage::create("msg-a", "Event", b"{}".to_vec()).unwrap(), - ); + ) + .await; store_message( &repo, OutboxMessage::create("msg-b", "Event", b"{}".to_vec()).unwrap(), - ); + ) + .await; store_message( &repo, OutboxMessage::create("msg-c", "Event", b"{}".to_vec()).unwrap(), - ); + ) + .await; let claimed = repo .outbox_store() @@ -679,14 +662,14 @@ mod tests { assert!(load_message(&repo, "msg-a").is_pending()); } - #[test] - fn claim_by_ids_skips_unclaimable_without_error() { + #[tokio::test] + async fn claim_by_ids_skips_unclaimable_without_error() { let repo = HashMapRepository::new(); let mut leased = OutboxMessage::create("msg-a", "Event", b"{}".to_vec()).unwrap(); leased .claim_for("other-worker", Duration::from_secs(60)) .unwrap(); - store_message(&repo, leased); + store_message(&repo, leased).await; // Requesting a currently-leased id (and a missing id) yields no claim, // not an error. @@ -723,11 +706,11 @@ mod tests { ); } - #[test] - fn competing_workers_only_claim_message_once() { + #[tokio::test] + async fn competing_workers_only_claim_message_once() { let repo = HashMapRepository::new(); let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let id = store_message(&repo, message); + let id = store_message(&repo, message).await; let barrier = Arc::new(Barrier::new(3)); let store_a = repo.outbox_store(); @@ -767,11 +750,11 @@ mod tests { assert_eq!(stored.attempts, 1); } - #[test] - fn publish_failure_releases_until_retry_ceiling_then_fails() { + #[tokio::test] + async fn publish_failure_releases_until_retry_ceiling_then_fails() { let repo = HashMapRepository::new(); let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let id = store_message(&repo, message); + let id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store @@ -827,11 +810,11 @@ mod tests { assert_eq!(store.fail(&claim, "error").unwrap_err(), expected); } - #[test] - fn stale_or_mismatched_claims_cannot_be_completed() { + #[tokio::test] + async fn stale_or_mismatched_claims_cannot_be_completed() { let repo = HashMapRepository::new(); let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let _id = store_message(&repo, message); + let _id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store @@ -850,18 +833,18 @@ mod tests { expired .claim_at("worker-1", Duration::from_secs(1), SystemTime::UNIX_EPOCH) .unwrap(); - let expired_id = store_message(&repo, expired); + let expired_id = store_message(&repo, expired).await; let expired = load_message(&repo, &expired_id); let claim = OutboxClaimRef::from_message(&expired).unwrap(); let err = store.complete(&claim).unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); } - #[test] - fn stale_attempt_claims_cannot_complete_later_claims() { + #[tokio::test] + async fn stale_attempt_claims_cannot_complete_later_claims() { let repo = HashMapRepository::new(); let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let _id = store_message(&repo, message); + let _id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store @@ -888,11 +871,11 @@ mod tests { store.complete(¤t_claim).unwrap(); } - #[test] - fn already_published_message_is_not_completed_again() { + #[tokio::test] + async fn already_published_message_is_not_completed_again() { let repo = HashMapRepository::new(); let message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); - let _id = store_message(&repo, message); + let _id = store_message(&repo, message).await; let store = repo.outbox_store(); let claimed = store diff --git a/src/read_model/in_memory.rs b/src/read_model/in_memory.rs index 1d0592450..3350bd27a 100644 --- a/src/read_model/in_memory.rs +++ b/src/read_model/in_memory.rs @@ -570,25 +570,6 @@ mod tests { RowPatch, }; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - fn test_row_schema() -> ReadModelSchema { ReadModelSchema { model_name: "TestRow".into(), @@ -602,24 +583,26 @@ mod tests { } } - #[test] - fn relational_write_plan_upserts_rows() { + #[tokio::test] + async fn relational_write_plan_upserts_rows() { let store = InMemoryReadModelStore::new(); let schema = test_row_schema(); let key = RowKey::new([("id", RowValue::String("row-1".into()))]); let mut values = RowValues::new(); values.insert("id", RowValue::String("row-1".into())); - let outcome = block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ - ReadModelMutation::UpsertRow(RowMutation { - schema: schema.clone(), - key: key.clone(), - values, - expected_version: ExpectedVersion::Any, - mode: RowWriteMode::Upsert, - }), - ]))) - .unwrap(); + let outcome = store + .commit_write_plan_async(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( + RowMutation { + schema: schema.clone(), + key: key.clone(), + values, + expected_version: ExpectedVersion::Any, + mode: RowWriteMode::Upsert, + }, + )])) + .await + .unwrap(); let row = store .relational_rows .read() @@ -636,34 +619,38 @@ mod tests { ); } - #[test] - fn relational_write_plan_patches_and_deletes_rows() { + #[tokio::test] + async fn relational_write_plan_patches_and_deletes_rows() { let store = InMemoryReadModelStore::new(); let schema = test_row_schema(); let key = RowKey::new([("id", RowValue::String("row-1".into()))]); let mut values = RowValues::new(); values.insert("id", RowValue::String("row-1".into())); - block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ - ReadModelMutation::UpsertRow(RowMutation { - schema: schema.clone(), - key: key.clone(), - values, - expected_version: ExpectedVersion::Any, - mode: RowWriteMode::Upsert, - }), - ]))) - .unwrap(); - block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ - ReadModelMutation::PatchRow(PatchRowMutation { - schema: schema.clone(), - key: key.clone(), - patch: RowPatch::new().set("id", RowValue::String("row-1".into())), - expected_version: ExpectedVersion::Exact(1), - mode: PatchMode::UpdateExisting, - }), - ]))) - .unwrap(); + store + .commit_write_plan_async(ReadModelWritePlan::new(vec![ReadModelMutation::UpsertRow( + RowMutation { + schema: schema.clone(), + key: key.clone(), + values, + expected_version: ExpectedVersion::Any, + mode: RowWriteMode::Upsert, + }, + )])) + .await + .unwrap(); + store + .commit_write_plan_async(ReadModelWritePlan::new(vec![ReadModelMutation::PatchRow( + PatchRowMutation { + schema: schema.clone(), + key: key.clone(), + patch: RowPatch::new().set("id", RowValue::String("row-1".into())), + expected_version: ExpectedVersion::Exact(1), + mode: PatchMode::UpdateExisting, + }, + )])) + .await + .unwrap(); let version = store .relational_rows .read() @@ -673,14 +660,16 @@ mod tests { .version; assert_eq!(version, 2); - block_on(store.commit_write_plan_async(ReadModelWritePlan::new(vec![ - ReadModelMutation::DeleteRow(DeleteRowMutation { - schema: schema.clone(), - key: key.clone(), - expected_version: ExpectedVersion::Exact(2), - }), - ]))) - .unwrap(); + store + .commit_write_plan_async(ReadModelWritePlan::new(vec![ReadModelMutation::DeleteRow( + DeleteRowMutation { + schema: schema.clone(), + key: key.clone(), + expected_version: ExpectedVersion::Exact(2), + }, + )])) + .await + .unwrap(); assert!(!store .relational_rows .read() diff --git a/src/snapshot/in_memory.rs b/src/snapshot/in_memory.rs index 1febd38e1..4ca73ff32 100644 --- a/src/snapshot/in_memory.rs +++ b/src/snapshot/in_memory.rs @@ -82,31 +82,12 @@ impl AsyncSnapshotStore for InMemorySnapshotStore { mod tests { use super::*; - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - fn identity(id: &str) -> StreamIdentity { StreamIdentity::new("test.aggregate", id).unwrap() } - #[test] - fn save_and_get() { + #[tokio::test] + async fn save_and_get() { let store = InMemorySnapshotStore::new(); let record = SnapshotRecord::new( "test.aggregate", @@ -116,9 +97,14 @@ mod tests { 1, vec![1, 2, 3], ); - block_on(store.save_snapshot_async(&identity("agg-1"), record)).unwrap(); + store + .save_snapshot_async(&identity("agg-1"), record) + .await + .unwrap(); - let loaded = block_on(store.get_snapshot_async(&identity("agg-1"))) + let loaded = store + .get_snapshot_async(&identity("agg-1")) + .await .unwrap() .unwrap(); assert_eq!(loaded.version, 5); @@ -126,66 +112,88 @@ mod tests { assert_eq!(loaded.snapshot_type, "TestSnapshot"); } - #[test] - fn get_missing_returns_none() { + #[tokio::test] + async fn get_missing_returns_none() { let store = InMemorySnapshotStore::new(); - assert!(block_on(store.get_snapshot_async(&identity("missing"))) + assert!(store + .get_snapshot_async(&identity("missing")) + .await .unwrap() .is_none()); } - #[test] - fn save_overwrites() { + #[tokio::test] + async fn save_overwrites() { let store = InMemorySnapshotStore::new(); - block_on(store.save_snapshot_async( - &identity("agg-1"), - SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), - )) - .unwrap(); - block_on(store.save_snapshot_async( - &identity("agg-1"), - SnapshotRecord::new("test.aggregate", "agg-1", 5, "TestSnapshot", 1, vec![5]), - )) - .unwrap(); - - let loaded = block_on(store.get_snapshot_async(&identity("agg-1"))) + store + .save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), + ) + .await + .unwrap(); + store + .save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 5, "TestSnapshot", 1, vec![5]), + ) + .await + .unwrap(); + + let loaded = store + .get_snapshot_async(&identity("agg-1")) + .await .unwrap() .unwrap(); assert_eq!(loaded.version, 5); assert_eq!(loaded.payload, vec![5]); } - #[test] - fn delete_existing() { + #[tokio::test] + async fn delete_existing() { let store = InMemorySnapshotStore::new(); - block_on(store.save_snapshot_async( - &identity("agg-1"), - SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), - )) - .unwrap(); - assert!(block_on(store.delete_snapshot_async(&identity("agg-1"))).unwrap()); - assert!(block_on(store.get_snapshot_async(&identity("agg-1"))) + store + .save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 1, "TestSnapshot", 1, vec![1]), + ) + .await + .unwrap(); + assert!(store + .delete_snapshot_async(&identity("agg-1")) + .await + .unwrap()); + assert!(store + .get_snapshot_async(&identity("agg-1")) + .await .unwrap() .is_none()); } - #[test] - fn delete_missing_returns_false() { + #[tokio::test] + async fn delete_missing_returns_false() { let store = InMemorySnapshotStore::new(); - assert!(!block_on(store.delete_snapshot_async(&identity("missing"))).unwrap()); + assert!(!store + .delete_snapshot_async(&identity("missing")) + .await + .unwrap()); } - #[test] - fn clone_shares_storage() { + #[tokio::test] + async fn clone_shares_storage() { let store = InMemorySnapshotStore::new(); let clone = store.clone(); - block_on(store.save_snapshot_async( - &identity("agg-1"), - SnapshotRecord::new("test.aggregate", "agg-1", 3, "TestSnapshot", 1, vec![3]), - )) - .unwrap(); + store + .save_snapshot_async( + &identity("agg-1"), + SnapshotRecord::new("test.aggregate", "agg-1", 3, "TestSnapshot", 1, vec![3]), + ) + .await + .unwrap(); - let loaded = block_on(clone.get_snapshot_async(&identity("agg-1"))) + let loaded = clone + .get_snapshot_async(&identity("agg-1")) + .await .unwrap() .unwrap(); assert_eq!(loaded.version, 3); diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index d42e750b9..07923b52a 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -371,27 +371,8 @@ mod tests { } } - fn block_on(future: F) -> F::Output { - use std::ptr; - use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; - const VTABLE: RawWakerVTable = RawWakerVTable::new( - |_| RawWaker::new(ptr::null(), &VTABLE), - |_| {}, - |_| {}, - |_| {}, - ); - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; - let mut cx = Context::from_waker(&waker); - let mut future = std::pin::pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { - return output; - } - } - } - - #[test] - fn snapshot_batch_failure_leaves_aggregate_uncommitted() { + #[tokio::test] + async fn snapshot_batch_failure_leaves_aggregate_uncommitted() { let repo = FailingSnapshotRepo::default(); let aggregate_repo = AsyncAggregateRepository::new(repo); let snapshot_repo = AsyncSnapshotAggregateRepository::new(aggregate_repo, 1); @@ -399,7 +380,7 @@ mod tests { let mut aggregate = TestAggregate::default(); aggregate.touch().unwrap(); - let err = block_on(snapshot_repo.commit(&mut aggregate)).unwrap_err(); + let err = snapshot_repo.commit(&mut aggregate).await.unwrap_err(); assert_eq!(err, RepositoryError::Model("snapshot write failed".into())); assert!(snapshot_repo diff --git a/tests/distributed_read_model_board/projections_service/handlers/board.rs b/tests/distributed_read_model_board/projections_service/handlers/board.rs index da87a4528..84942b538 100644 --- a/tests/distributed_read_model_board/projections_service/handlers/board.rs +++ b/tests/distributed_read_model_board/projections_service/handlers/board.rs @@ -29,7 +29,7 @@ pub async fn handle(ctx: &Context<'_, ProjectionDependencies>) -> Result BoardView { /// The aggregate version is the trailing segment of the outbox event id /// (`::`). -fn event_version(id: &str) -> i64 { - id.rsplit(':') - .next() - .expect("board projection event id should include a version segment") - .parse() - .expect("board projection event id should end with a numeric aggregate version") +fn event_version(id: &str) -> Result { + let segment = id.rsplit(':').next().ok_or_else(|| { + HandlerError::DecodeFailed( + "board projection event id should include a version segment".into(), + ) + })?; + segment.parse().map_err(|_| { + HandlerError::DecodeFailed( + "board projection event id should end with a numeric aggregate version".into(), + ) + }) } #[cfg(test)] @@ -99,14 +104,12 @@ mod tests { #[test] fn event_version_parses_trailing_outbox_segment() { - assert_eq!(event_version("board-1:board.card_added:42"), 42); + assert_eq!(event_version("board-1:board.card_added:42").unwrap(), 42); } #[test] - #[should_panic( - expected = "board projection event id should end with a numeric aggregate version" - )] - fn event_version_panics_on_malformed_outbox_segment() { - event_version("board-1:board.card_added:bad"); + fn event_version_rejects_malformed_outbox_segment() { + let err = event_version("board-1:board.card_added:bad").unwrap_err(); + assert!(matches!(err, HandlerError::DecodeFailed(_))); } } diff --git a/tests/todos/main.rs b/tests/todos/main.rs index 618e6a096..a444f9ced 100644 --- a/tests/todos/main.rs +++ b/tests/todos/main.rs @@ -143,15 +143,17 @@ async fn todos() { .unwrap(); // Commit multiple Todos to the repository - let _ = repo.commit_all(&mut [&mut todo2, &mut todo3]).await; + repo.commit_all(&mut [&mut todo2, &mut todo3]) + .await + .expect("bulk commit should succeed"); // get all the todos from the repository let all_todos = repo.peek_all(&[&id1, &id2, &id3]).await.unwrap(); - if !all_todos.is_empty() { - assert!(all_todos.len() == 3); - } else { - println!("No Todos found"); - } + assert_eq!( + all_todos.len(), + 3, + "expected all committed todos to be present" + ); } #[tokio::test]