diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 892082d96c6..b53fb54fa43 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -543,6 +543,14 @@ Note: Both `TriggerDef` and `ActionDef` use serde internally-tagged enums. Trigg | `request_approval` | Suspend execution; fields: `from`, `message`, `timeout` (default 24h) | | `delay` | Pause execution (max 300 seconds) | +**Workflow-to-agent authority:** A workflow `send_message` action persists one ordinary relay-signed kind `9` timeline message containing the rendered text, then creates one durable delivery row per owner-signed target. The visible message is identified by a single `["buzz:workflow", "message-v1"]` marker and binds the exact kind `30620` definition revision, run, step, typed cause (`event`, `command`, `schedule`, or `webhook`), channel, and owner. Mention targets are derived only from the owner-signed template, never from rendered trigger or webhook data; `p` tags are routing metadata and never grant authority. + +The relay publishes an ephemeral kind `24620` wake containing the durable delivery ID and immutable binding fields. ACP accepts workflow authority only after authenticating that wake as relay-signed, atomically claiming its target-specific delivery lease, and verifying the referenced visible message and signed definition against the delivery's immutable execution snapshot. It reconstructs the rendered text from the exact stored trigger context and prior-step trace and requires byte-for-byte equality with the persisted timeline content. The durable `(run, step, target)` identity collapses retries while preserving distinct `send_message` steps from the same run. Lease renewal and fenced terminal acknowledgement provide reconnect and crash recovery without opening a second authority path. + +Webhook data remains untrusted and size-limited at ingress. It may affect rendered text only through slots declared by the signed definition; it is retained in the private delivery snapshot rather than exposed as transport payload. An unclaimed workflow-marked kind `9`, a malformed or forged wake, a mismatched definition/run/step/channel/message binding, missing relay identity, or rendering mismatch fails closed. A verified workflow owner remains subject to the same `respond_to` policy and DM hardening as a directly authored message. + +**Partial rollout:** durable workflow-to-agent delivery is producer-gated by `BUZZ_WORKFLOW_AGENT_DELIVERY_ENABLED`, which defaults to `false`. Operators upgrade all ACP harnesses first, then enable the relay producer; while disabled, `send_message` fails before publishing either the visible kind `9` or durable rows. This fence is required because a legacy ACP configured `respond-to=anyone` would otherwise execute a relay-authored kind `9` without claiming it. Once enabled, upgraded ACP drops any workflow-marked kind `9` that has not arrived through a successfully claimed durable wake. Ordinary messages are unaffected. + **Template variables:** `{{trigger.text}}`, `{{trigger.author}}`, `{{steps.ID.output.FIELD}}`. Single-pass resolution (not recursive). Unknown variables left as literal text. **Condition evaluation:** `evalexpr` with `HashMapContext`. Dot notation converted to underscores (`trigger.text` → `trigger_text`). Custom functions registered: `str_contains`, `str_starts_with`, `str_ends_with`, `str_len`. 100ms timeout prevents adversarial expressions from blocking. diff --git a/Cargo.lock b/Cargo.lock index 18c53c18ca0..9a2ac1fd054 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -833,6 +833,7 @@ dependencies = [ "buzz-core", "buzz-persona", "buzz-sdk", + "buzz-workflow", "chrono", "clap", "evalexpr", diff --git a/crates/buzz-acp/Cargo.toml b/crates/buzz-acp/Cargo.toml index d047849806f..cd0187e4ac5 100644 --- a/crates/buzz-acp/Cargo.toml +++ b/crates/buzz-acp/Cargo.toml @@ -19,6 +19,7 @@ path = "src/main.rs" # Internal buzz-core = { workspace = true } buzz-sdk = { workspace = true } +buzz-workflow = { workspace = true } buzz-persona = { path = "../buzz-persona" } # Nostr diff --git a/crates/buzz-acp/src/config.rs b/crates/buzz-acp/src/config.rs index 4a82cf6306d..3888cee86c2 100644 --- a/crates/buzz-acp/src/config.rs +++ b/crates/buzz-acp/src/config.rs @@ -1277,7 +1277,8 @@ pub fn resolve_channel_filters( rules: &[SubscriptionRule], ) -> HashMap { use buzz_core::kind::{ - KIND_STREAM_MESSAGE, KIND_STREAM_REMINDER, KIND_WORKFLOW_APPROVAL_REQUESTED, + KIND_STREAM_MESSAGE, KIND_STREAM_REMINDER, KIND_WORKFLOW_AGENT_WAKE, + KIND_WORKFLOW_APPROVAL_REQUESTED, }; let target_channels: Vec = if let Some(ref overrides) = config.channels_override { @@ -1297,6 +1298,7 @@ pub fn resolve_channel_filters( let kinds = config.kinds_override.clone().unwrap_or_else(|| { vec![ KIND_STREAM_MESSAGE, + KIND_WORKFLOW_AGENT_WAKE, KIND_WORKFLOW_APPROVAL_REQUESTED, KIND_STREAM_REMINDER, ] @@ -1379,7 +1381,8 @@ pub fn resolve_dynamic_channel_filter( rules: &[crate::filter::SubscriptionRule], ) -> Option { use buzz_core::kind::{ - KIND_STREAM_MESSAGE, KIND_STREAM_REMINDER, KIND_WORKFLOW_APPROVAL_REQUESTED, + KIND_STREAM_MESSAGE, KIND_STREAM_REMINDER, KIND_WORKFLOW_AGENT_WAKE, + KIND_WORKFLOW_APPROVAL_REQUESTED, }; // In Mentions/All mode, if the operator explicitly constrained channels @@ -1402,6 +1405,7 @@ pub fn resolve_dynamic_channel_filter( kinds: Some(config.kinds_override.clone().unwrap_or_else(|| { vec![ KIND_STREAM_MESSAGE, + KIND_WORKFLOW_AGENT_WAKE, KIND_WORKFLOW_APPROVAL_REQUESTED, KIND_STREAM_REMINDER, ] diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 146214197a8..eb74ccea891 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -22,7 +22,7 @@ use acp::{AcpClient, EnvVar, McpServer}; use anyhow::{ensure, Context, Result}; use buzz_core::kind::{ KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_STREAM_MESSAGE, - KIND_STREAM_REMINDER, KIND_WORKFLOW_APPROVAL_REQUESTED, + KIND_STREAM_REMINDER, KIND_WORKFLOW_AGENT_WAKE, KIND_WORKFLOW_APPROVAL_REQUESTED, }; use buzz_core::observer::{ decrypt_observer_payload, encrypt_observer_payload, OBSERVER_FRAME_TELEMETRY, @@ -65,6 +65,20 @@ const MODELS_TIMEOUT: Duration = Duration::from_secs(10); /// Timeout for `buzz-acp authenticate`. Browser-based vendor auth can require /// human interaction, so it must not share the short probe timeout. const AUTHENTICATE_TIMEOUT: Duration = Duration::from_secs(10 * 60); +const WORKFLOW_LEASE_RETRY_MARGIN_SECS: u64 = 400; +const WORKFLOW_LEASE_RENEW_INTERVAL_SECS: u64 = 30; +const WORKFLOW_LEASE_FAIL_CLOSED_MARGIN_SECS: i64 = 60; + +fn workflow_lease_seconds(config: &Config) -> i64 { + config + .max_turn_duration_secs + .saturating_add(WORKFLOW_LEASE_RETRY_MARGIN_SECS) as i64 +} + +const _: () = assert!( + config::MAX_TURN_DURATION_CEILING_SECS + WORKFLOW_LEASE_RETRY_MARGIN_SECS + <= buzz_core::workflow_delivery::MAX_LEASE_SECONDS as u64 +); /// Resolve the process working directory for ACP session metadata and prompts. /// @@ -248,6 +262,615 @@ async fn is_owner_or_sibling( /// siblings may fire a turn — the explicit allowlist and `anyone` mode do /// NOT apply inside DMs. `Nobody` still drops everything. Callers must /// resolve `is_dm` fail-closed: unknown channel type ⇒ treat as DM. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +struct WorkflowDeliveryBinding { + run_id: Uuid, + step_id: String, + definition_event_id: String, + message_event_id: String, + channel_id: Uuid, +} + +#[derive(Clone, Debug, serde::Deserialize)] +struct ClaimedWorkflowDelivery { + id: Uuid, + workflow_id: Uuid, + run_id: Uuid, + step_id: String, + definition_event_id: String, + message_event_id: String, + channel_id: Uuid, + target_pubkey: String, + attempt: i32, + claim_token: Uuid, + claim_expires_at: chrono::DateTime, + execution_trace: serde_json::Value, + trigger_context: Option, +} + +async fn claim_workflow_delivery( + rest_client: &relay::RestClient, + delivery_id: Option, + expected: Option<&WorkflowDeliveryBinding>, + lease_seconds: i64, +) -> Option { + let response = rest_client + .post_json( + "/workflows/agent-deliveries/claim", + &serde_json::json!({ + "delivery_id": delivery_id, + "expected": expected, + "lease_seconds": lease_seconds, + }), + ) + .await + .ok()?; + let delivery: ClaimedWorkflowDelivery = + serde_json::from_value(response.get("delivery")?.clone()).ok()?; + tracing::debug!( + delivery = %delivery.id, + attempt = delivery.attempt, + "claimed workflow delivery" + ); + Some(delivery) +} + +async fn renew_workflow_delivery( + rest_client: &relay::RestClient, + delivery: &ClaimedWorkflowDelivery, + lease_seconds: i64, +) -> Option> { + let path = format!("/workflows/agent-deliveries/{}/renew", delivery.id); + let response = rest_client + .post_json( + &path, + &serde_json::json!({ + "claim_token": delivery.claim_token, + "lease_seconds": lease_seconds, + }), + ) + .await + .ok()?; + serde_json::from_value(response.get("claim_expires_at")?.clone()).ok() +} + +#[allow(clippy::too_many_arguments)] +async fn renew_owned_workflow_deliveries( + rest_client: &relay::RestClient, + lease_seconds: i64, + pool: &mut AgentPool, + queue: &mut EventQueue, + workflow_deliveries_by_event: &mut HashMap, + workflow_deliveries_by_turn: &mut HashMap>, + workflow_native_steer_turn_by_event: &mut HashMap, + completed_workflow_turns: &mut HashMap, + pending_workflow_finalizations: &mut HashMap, +) { + #[derive(Clone)] + enum Owner { + Event(String), + Turn(String), + Finalizing(Uuid), + } + + let mut owned = workflow_deliveries_by_event + .iter() + .map(|(event_id, delivery)| (Owner::Event(event_id.clone()), delivery.clone())) + .collect::>(); + owned.extend( + workflow_deliveries_by_turn + .iter() + .flat_map(|(turn_id, deliveries)| { + deliveries + .iter() + .cloned() + .map(|delivery| (Owner::Turn(turn_id.clone()), delivery)) + .collect::>() + }), + ); + owned.extend( + pending_workflow_finalizations + .iter() + .map(|(delivery_id, finalization)| { + ( + Owner::Finalizing(*delivery_id), + finalization.delivery.clone(), + ) + }), + ); + + // All claims renew concurrently under one aggregate deadline. The main + // loop never spends one REST retry budget per claim while later leases + // silently expire behind it. + let renewals = owned.iter().map(|(_, delivery)| { + let delivery = delivery.clone(); + async move { + let renewed = tokio::time::timeout( + Duration::from_secs(WORKFLOW_LEASE_FAIL_CLOSED_MARGIN_SECS as u64 / 2), + renew_workflow_delivery(rest_client, &delivery, lease_seconds), + ) + .await + .ok() + .flatten(); + (delivery, renewed) + } + }); + let results = futures_util::future::join_all(renewals).await; + + for ((owner, _), (delivery, renewed)) in owned.into_iter().zip(results) { + if let Some(claim_expires_at) = renewed { + match &owner { + Owner::Event(event_id) => { + if let Some(current) = workflow_deliveries_by_event.get_mut(event_id) { + current.claim_expires_at = claim_expires_at; + } + } + Owner::Turn(turn_id) => { + if let Some(current) = + workflow_deliveries_by_turn + .get_mut(turn_id) + .and_then(|deliveries| { + deliveries.iter_mut().find(|item| item.id == delivery.id) + }) + { + current.claim_expires_at = claim_expires_at; + } + } + Owner::Finalizing(delivery_id) => { + if let Some(current) = pending_workflow_finalizations.get_mut(delivery_id) { + current.delivery.claim_expires_at = claim_expires_at; + } + } + } + continue; + } + + let seconds_left = (delivery.claim_expires_at - chrono::Utc::now()).num_seconds(); + tracing::warn!(delivery = %delivery.id, seconds_left, "workflow delivery lease renewal failed"); + if seconds_left > WORKFLOW_LEASE_FAIL_CLOSED_MARGIN_SECS { + continue; + } + + tracing::error!(delivery = %delivery.id, channel = %delivery.channel_id, + "workflow delivery lease cannot be maintained — stopping local ownership"); + signal_in_flight_task( + pool, + delivery.channel_id, + ControlSignal::TerminalWorkflowCancel, + ); + queue.terminalize_workflow_event(delivery.channel_id, &delivery.message_event_id); + match owner { + Owner::Event(event_id) => { + workflow_deliveries_by_event.remove(&event_id); + if let Some(turn_id) = workflow_native_steer_turn_by_event.remove(&event_id) { + clear_completed_workflow_turn_if_resolved( + &turn_id, + workflow_native_steer_turn_by_event, + completed_workflow_turns, + ); + } + } + Owner::Turn(turn_id) => { + if let Some(deliveries) = workflow_deliveries_by_turn.get_mut(&turn_id) { + deliveries.retain(|item| item.id != delivery.id); + if deliveries.is_empty() { + workflow_deliveries_by_turn.remove(&turn_id); + } + } + } + Owner::Finalizing(delivery_id) => { + // Never replace an uncertain terminal disposition with a + // different failure. An exact finish replay may already have + // committed while its response was lost; preserve that intent + // for reconciliation until the bounded owner window closes. + tracing::error!( + delivery = %delivery_id, + "workflow finalization lease cannot be renewed" + ); + continue; + } + } + finish_workflow_delivery( + rest_client, + pending_workflow_finalizations, + &delivery, + false, + false, + Some("lease_renewal_failed"), + Some("managed agent stopped because exclusive delivery ownership could not be renewed"), + ) + .await; + } +} + +async fn verified_workflow_delivery_message( + delivery: &ClaimedWorkflowDelivery, + rest_client: &relay::RestClient, + agent_pubkey: &str, + relay_self: Option<&str>, +) -> Option<(nostr::Event, String)> { + use buzz_workflow::executor::{resolve_template, TriggerContext}; + use buzz_workflow::schema::ActionDef; + + let definition_id = nostr::EventId::from_hex(&delivery.definition_event_id).ok()?; + let message_id = nostr::EventId::from_hex(&delivery.message_event_id).ok()?; + let definition = query_exact_event(definition_id, rest_client).await?; + let message = query_exact_event(message_id, rest_client).await?; + if !delivery.target_pubkey.eq_ignore_ascii_case(agent_pubkey) + || definition.kind.as_u16() as u32 != buzz_core::kind::KIND_WORKFLOW_DEF + || workflow_uuid(&definition)? != delivery.workflow_id + || !definition_channel_matches(&definition, delivery.channel_id) + || message.kind.as_u16() as u32 != KIND_STREAM_MESSAGE + || !event_channel_matches(&message, delivery.channel_id) + || !relay_self.is_some_and(|relay| message.pubkey.to_hex().eq_ignore_ascii_case(relay)) + || !exact_tags(&message, "p").iter().any(|tag| { + tag.as_slice() + .get(1) + .is_some_and(|value| value.eq_ignore_ascii_case(agent_pubkey)) + }) + || !exact_tags(&message, "workflow-definition") + .iter() + .any(|tag| { + tag.as_slice() + .get(1) + .is_some_and(|value| value.eq_ignore_ascii_case(&delivery.definition_event_id)) + }) + || !exact_tags(&message, "workflow-run").iter().any(|tag| { + tag.as_slice() + .get(1) + .is_some_and(|value| value == &delivery.run_id.to_string()) + }) + || !exact_tags(&message, "workflow-step").iter().any(|tag| { + tag.as_slice() + .get(1) + .is_some_and(|value| value == &delivery.step_id) + }) + { + return None; + } + let (workflow, _) = buzz_workflow::WorkflowEngine::parse_yaml(&definition.content).ok()?; + let step = workflow + .steps + .iter() + .find(|step| step.id == delivery.step_id)?; + let ActionDef::SendMessage { text, channel, .. } = &step.action else { + return None; + }; + if channel + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .is_some_and(|value| value.parse::().ok() != Some(delivery.channel_id)) + { + return None; + } + let trigger: TriggerContext = serde_json::from_value(delivery.trigger_context.clone()?).ok()?; + if !trigger + .definition_event_id + .eq_ignore_ascii_case(&delivery.definition_event_id) + { + return None; + } + let mut outputs = HashMap::new(); + for trace in delivery.execution_trace.as_array()? { + let step_id = trace.get("step_id")?.as_str()?; + let output = trace.get("output")?.clone(); + outputs.insert(step_id.to_string(), output); + } + let rendered = resolve_template(text, &trigger, &outputs).ok()?; + if message.content != rendered { + return None; + } + Some((message, definition.pubkey.to_hex())) +} + +fn retain_workflow_deliveries_for_local_retry( + queue: &EventQueue, + channel_id: Option, + allow_local_retry: bool, + deliveries: Vec, + workflow_deliveries_by_event: &mut HashMap, +) -> Vec { + let mut terminal = Vec::new(); + for delivery in deliveries { + if allow_local_retry + && channel_id.is_some_and(|channel_id| { + queue.contains_event(channel_id, &delivery.message_event_id) + }) + { + workflow_deliveries_by_event.insert(delivery.message_event_id.clone(), delivery); + } else { + terminal.push(delivery); + } + } + terminal +} + +async fn try_finish_workflow_delivery( + rest_client: &relay::RestClient, + delivery: &ClaimedWorkflowDelivery, + delivered: bool, + retryable: bool, + failure_code: Option<&str>, + failure_message: Option<&str>, +) -> bool { + let path = format!("/workflows/agent-deliveries/{}/finish", delivery.id); + let result = rest_client + .post_json( + &path, + &serde_json::json!({ + "claim_token": delivery.claim_token, + "delivered": delivered, + "retryable": retryable, + "failure_code": failure_code, + "failure_message": failure_message, + }), + ) + .await; + match result { + Ok(_) => true, + Err(error) => { + tracing::warn!(delivery = %delivery.id, %error, "workflow delivery finish failed"); + false + } + } +} + +#[derive(Clone)] +struct PendingWorkflowFinalization { + delivery: ClaimedWorkflowDelivery, + delivered: bool, + retryable: bool, + failure_code: Option, + failure_message: Option, +} + +async fn finish_workflow_delivery( + rest_client: &relay::RestClient, + pending: &mut HashMap, + delivery: &ClaimedWorkflowDelivery, + delivered: bool, + retryable: bool, + failure_code: Option<&str>, + failure_message: Option<&str>, +) { + let finalization = PendingWorkflowFinalization { + delivery: delivery.clone(), + delivered, + retryable, + failure_code: failure_code.map(str::to_owned), + failure_message: failure_message.map(str::to_owned), + }; + pending.insert(delivery.id, finalization); + // Delivery work and its terminal intent are recorded synchronously, but + // network I/O is deferred to the concurrent reconciliation stage. This + // keeps result, panic, channel-removal, and shutdown loops from spending a + // serial REST retry budget while claims later in the ledger go unrenewed. + let _ = rest_client; +} + +async fn reconcile_workflow_finalizations( + rest_client: &relay::RestClient, + pending: &mut HashMap, +) { + let now = chrono::Utc::now(); + let attempts = pending.values().filter_map(|finalization| { + let safe_seconds = (finalization.delivery.claim_expires_at - now).num_seconds() + - WORKFLOW_LEASE_FAIL_CLOSED_MARGIN_SECS; + if safe_seconds <= 0 { + tracing::error!( + delivery = %finalization.delivery.id, + "workflow finalization has no safe I/O window" + ); + return None; + } + let finalization = finalization.clone(); + Some(async move { + let delivery_id = finalization.delivery.id; + let finished = tokio::time::timeout( + Duration::from_secs(safe_seconds.min(30) as u64), + try_finish_workflow_delivery( + rest_client, + &finalization.delivery, + finalization.delivered, + finalization.retryable, + finalization.failure_code.as_deref(), + finalization.failure_message.as_deref(), + ), + ) + .await + .unwrap_or(false); + (delivery_id, finished) + }) + }); + for (delivery_id, finished) in futures_util::future::join_all(attempts).await { + if finished { + pending.remove(&delivery_id); + } + } +} + +async fn transfer_native_steer_workflow_delivery( + rest_client: &relay::RestClient, + event_id: &str, + turn_id: &str, + workflow_deliveries_by_event: &mut HashMap, + workflow_deliveries_by_turn: &mut HashMap>, + completed_workflow_turns: &mut HashMap, + pending_workflow_finalizations: &mut HashMap, +) { + let Some(delivery) = workflow_deliveries_by_event.remove(event_id) else { + return; + }; + if let Some(delivered) = completed_workflow_turns.get(turn_id).copied() { + finish_workflow_delivery( + rest_client, + pending_workflow_finalizations, + &delivery, + delivered, + false, + (!delivered).then_some("prompt_failed"), + (!delivered).then_some("managed agent turn terminated without a local retry"), + ) + .await; + } else { + workflow_deliveries_by_turn + .entry(turn_id.to_owned()) + .or_default() + .push(delivery); + } +} + +fn clear_completed_workflow_turn_if_resolved( + turn_id: &str, + workflow_native_steer_turn_by_event: &HashMap, + completed_workflow_turns: &mut HashMap, +) { + if !workflow_native_steer_turn_by_event + .values() + .any(|pending_turn| pending_turn == turn_id) + { + completed_workflow_turns.remove(turn_id); + } +} + +fn exact_single_tag_value<'a>(event: &'a nostr::Event, name: &str) -> Option<&'a str> { + let tags = exact_tags(event, name); + (tags.len() == 1 && tags[0].as_slice().len() == 2).then(|| tags[0].as_slice()[1].as_str()) +} + +/// Authenticate and parse a relay-authored live wake before claiming. Offline +/// polling is the only path allowed to claim without an exact delivery id. +fn trusted_live_workflow_wake_delivery_id( + event: &nostr::Event, + channel_id: Uuid, + agent_pubkey: &str, + relay_self: Option<&str>, +) -> Option<(Uuid, WorkflowDeliveryBinding)> { + let relay_self = relay_self?; + if event.kind.as_u16() as u32 != KIND_WORKFLOW_AGENT_WAKE + || !event.pubkey.to_hex().eq_ignore_ascii_case(relay_self) + || event.verify().is_err() + || !exact_single_tag_value(event, "p")?.eq_ignore_ascii_case(agent_pubkey) + || exact_single_tag_value(event, "h")?.parse::().ok()? != channel_id + { + return None; + } + nostr::PublicKey::from_hex(exact_single_tag_value(event, "p")?).ok()?; + nostr::EventId::from_hex(exact_single_tag_value(event, "workflow-definition")?).ok()?; + nostr::EventId::from_hex(exact_single_tag_value(event, "message")?).ok()?; + let run_id = exact_single_tag_value(event, "workflow-run")? + .parse::() + .ok()?; + let step_id = exact_single_tag_value(event, "workflow-step")?; + if step_id.is_empty() { + return None; + } + let delivery_id = exact_single_tag_value(event, "delivery")? + .parse::() + .ok()?; + Some(( + delivery_id, + WorkflowDeliveryBinding { + run_id, + step_id: step_id.to_owned(), + definition_event_id: exact_single_tag_value(event, "workflow-definition")?.to_owned(), + message_event_id: exact_single_tag_value(event, "message")?.to_owned(), + channel_id, + }, + )) +} + +fn wake_references_delivery( + event: &nostr::Event, + channel_id: Uuid, + agent_pubkey: &str, + relay_self: Option<&str>, + delivery: &ClaimedWorkflowDelivery, +) -> bool { + trusted_live_workflow_wake_delivery_id(event, channel_id, agent_pubkey, relay_self) + .is_some_and(|(delivery_id, _)| delivery_id == delivery.id) + && exact_single_tag_value(event, "workflow-definition") + .is_some_and(|v| v.eq_ignore_ascii_case(&delivery.definition_event_id)) + && exact_single_tag_value(event, "workflow-run") + .is_some_and(|v| v == delivery.run_id.to_string()) + && exact_single_tag_value(event, "workflow-step").is_some_and(|v| v == delivery.step_id) + && exact_single_tag_value(event, "message") + .is_some_and(|v| v.eq_ignore_ascii_case(&delivery.message_event_id)) + && exact_single_tag_value(event, "p") + .is_some_and(|v| v.eq_ignore_ascii_case(&delivery.target_pubkey)) + && delivery.target_pubkey.eq_ignore_ascii_case(agent_pubkey) + && exact_single_tag_value(event, "h").is_some_and(|v| v == delivery.channel_id.to_string()) + && delivery.channel_id == channel_id +} +fn exact_tags<'a>(event: &'a nostr::Event, name: &str) -> Vec<&'a nostr::Tag> { + event + .tags + .iter() + .filter(|tag| tag.as_slice().first().map(String::as_str) == Some(name)) + .collect() +} + +fn workflow_delivery_principal( + author: &str, + durable_workflow_owner: Option<&str>, + workflow_shape: bool, +) -> Option { + if durable_workflow_owner.is_none() && workflow_shape { + return None; + } + Some( + durable_workflow_owner + .map(str::to_owned) + .unwrap_or_else(|| author.to_owned()), + ) +} + +fn is_workflow_delivery_candidate(event: &nostr::Event, relay_self: Option<&str>) -> bool { + relay_self.is_some_and(|relay| { + event.kind.as_u16() as u32 == buzz_core::kind::KIND_STREAM_MESSAGE + && event.pubkey.to_hex().eq_ignore_ascii_case(relay) + && !exact_tags(event, "buzz:workflow").is_empty() + }) +} + +async fn query_exact_event( + id: nostr::EventId, + rest_client: &relay::RestClient, +) -> Option { + let response = tokio::time::timeout( + Duration::from_millis(2000), + rest_client.query(&[nostr::Filter::new().id(id)]), + ) + .await + .ok()? + .ok()?; + let events = response.as_array()?; + if events.len() != 1 { + return None; + } + let event = serde_json::from_value::(events[0].clone()).ok()?; + (event.id == id && event.verify().is_ok()).then_some(event) +} + +fn event_channel_matches(event: &nostr::Event, channel_id: Uuid) -> bool { + let channels = exact_tags(event, "h"); + channels.len() == 1 + && channels[0].as_slice().len() == 2 + && channels[0].as_slice()[1].parse::().ok() == Some(channel_id) +} + +fn definition_channel_matches(definition: &nostr::Event, channel_id: Uuid) -> bool { + event_channel_matches(definition, channel_id) +} + +fn workflow_uuid(definition: &nostr::Event) -> Option { + let tags = exact_tags(definition, "d"); + (tags.len() == 1 && tags[0].as_slice().len() == 2) + .then(|| tags[0].as_slice()[1].parse().ok()) + .flatten() +} + async fn author_allowed( respond_to: &RespondTo, allowlist: &HashSet, @@ -1567,6 +2190,8 @@ struct RespawnResult { struct SteerAckEvent { channel_id: Uuid, event_id: String, + /// Exact prompt turn that accepted the steer request. + turn_id: String, /// `Ok` if the read loop sent any of the locked `SteerAck` variants. /// `Err` if the oneshot was dropped without a send — should not happen /// under the current read-loop drains, but if it ever does the main @@ -2007,6 +2632,8 @@ async fn tokio_main() -> Result<()> { .await .map_err(|e| anyhow::anyhow!("relay connect error: {e}"))?; + let relay_self = relay.relay_self().map(str::to_owned); + // Tell the relay background task the watermark so it can use // `since = watermark - 5s` on the first REQ instead of `since=now`. // Best-effort: a failure here is non-fatal (we just lose the startup window @@ -2250,6 +2877,11 @@ async fn tokio_main() -> Result<()> { } else { None }; + // Poll durable workflow handoffs independently of ephemeral wakes. The + // immediate first tick covers startup; subsequent ticks cover missed wakes + // and reconnects. + let mut workflow_delivery_poll = tokio::time::interval(Duration::from_secs(15)); + workflow_delivery_poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut typing_refresh = if config.typing_enabled { let interval = Duration::from_secs(3); @@ -2304,6 +2936,7 @@ async fn tokio_main() -> Result<()> { // spawn_and_init never blocks the main loop. let maintenance_interval = Duration::from_secs(30); let mut last_maintenance = std::time::Instant::now(); + let mut last_workflow_lease_renewal = std::time::Instant::now(); // Channel for background respawn tasks to return completed agents. // Bounded to agent count — at most one respawn per slot in flight. @@ -2358,6 +2991,22 @@ async fn tokio_main() -> Result<()> { // Rotates at 1000 entries instead of clearing the entire set at 2000. let mut seen_membership_current: HashSet = HashSet::new(); let mut seen_membership_previous: HashSet = HashSet::new(); + // Durable workflow leases are indexed by the turn they enter. The ephemeral + // wake is only discovery; completion is fenced by the claim token after the + // actual prompt outcome returns. + let mut workflow_deliveries_by_event: HashMap = HashMap::new(); + let mut workflow_deliveries_by_turn: HashMap> = + HashMap::new(); + // Terminal dispositions remain owned until the relay authoritatively + // acknowledges the fenced finish. This prevents a lost HTTP response from + // turning successful work into a future redelivery. + let mut pending_workflow_finalizations: HashMap = + HashMap::new(); + // Native steers retain durable delivery ownership on the queued event until + // the agent acknowledges success. Results that race ahead of that ack are + // retained so success can finish against the exact fenced turn outcome. + let mut workflow_native_steer_turn_by_event: HashMap = HashMap::new(); + let mut completed_workflow_turns: HashMap = HashMap::new(); // Channels the agent has been removed from. When a checked-out agent is // returned to the pool, its sessions for these channels are stripped, and @@ -2438,6 +3087,26 @@ async fn tokio_main() -> Result<()> { } } + if last_workflow_lease_renewal.elapsed() + >= Duration::from_secs(WORKFLOW_LEASE_RENEW_INTERVAL_SECS) + { + last_workflow_lease_renewal = std::time::Instant::now(); + renew_owned_workflow_deliveries( + &ctx.rest_client, + workflow_lease_seconds(&config), + &mut pool, + &mut queue, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + &mut workflow_native_steer_turn_by_event, + &mut completed_workflow_turns, + &mut pending_workflow_finalizations, + ) + .await; + reconcile_workflow_finalizations(&ctx.rest_client, &mut pending_workflow_finalizations) + .await; + } + if pool_ready && last_maintenance.elapsed() >= maintenance_interval { last_maintenance = std::time::Instant::now(); queue.compact_expired_state(); @@ -2472,9 +3141,14 @@ async fn tokio_main() -> Result<()> { // called on relay events or pool results, neither of which // arrive when the channel is silent. if queue.has_flushable_work() { - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } @@ -2524,9 +3198,14 @@ async fn tokio_main() -> Result<()> { // this, batches requeued during crash recovery sit idle until the // next relay event arrives — which can be minutes on quiet channels. if respawn_collected { - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } @@ -2624,11 +3303,155 @@ async fn tokio_main() -> Result<()> { None } // Remaining branches don't touch pool — evaluated when pool is idle. + _ = workflow_delivery_poll.tick() => { + let _ = result_rx; + if let Some(delivery) = claim_workflow_delivery( + &ctx.rest_client, + None, + None, + workflow_lease_seconds(&config), + ).await { + match verified_workflow_delivery_message( + &delivery, + &ctx.rest_client, + &pubkey_hex, + relay_self.as_deref(), + ).await { + Some((message, owner)) => { + let channel_id = delivery.channel_id; + let is_dm = is_dm_channel(channel_id, &ctx.channel_info).await; + let allowed = author_allowed( + &config.respond_to, + &config.respond_to_allowlist, + &owner, + is_dm, + &owner_cache, + &ctx.rest_client, + ).await; + let matched = if allowed { + filter::match_event(&message, channel_id, &rules, &pubkey_hex).await + } else { + None + }; + if let Some(matched) = matched { + let event_id = message.id.to_hex(); + if queue.push(QueuedEvent { + channel_id, + event: message, + received_at: std::time::Instant::now(), + prompt_tag: matched.prompt_tag, + }) { + workflow_deliveries_by_event.insert(event_id, delivery); + } + if pool_ready { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { + typing_channels.insert(channel_id, thread_tags); + } + } + } else { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("delivery_not_admitted"), + Some("verified workflow message did not pass ACP admission"), + ).await; + } + } + None => { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("delivery_verification_failed"), + Some("definition, execution state, or visible message did not verify"), + ).await; + } + } + } + None + } buzz_event = relay.next_event() => { let _ = result_rx; // end split borrow before relay handling match buzz_event { - Some(buzz_event) => { - let kind_u32 = buzz_event.event.kind.as_u16() as u32; + Some(mut buzz_event) => { + let mut kind_u32 = buzz_event.event.kind.as_u16() as u32; + let mut durable_workflow_owner: Option = None; + let mut claimed_workflow_delivery: Option = + None; + + if kind_u32 == KIND_WORKFLOW_AGENT_WAKE { + let wake = buzz_event.event.clone(); + let wake_channel_id = buzz_event.channel_id; + let Some((wake_delivery_id, wake_binding)) = trusted_live_workflow_wake_delivery_id( + &wake, + wake_channel_id, + &pubkey_hex, + relay_self.as_deref(), + ) else { + tracing::warn!(event = %wake.id, "discarding untrusted workflow wake"); + continue; + }; + let Some(delivery) = claim_workflow_delivery( + &ctx.rest_client, + Some(wake_delivery_id), + Some(&wake_binding), + workflow_lease_seconds(&config), + ).await else { + continue; + }; + if !wake_references_delivery( + &wake, + wake_channel_id, + &pubkey_hex, + relay_self.as_deref(), + &delivery, + ) { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("wake_binding_mismatch"), + Some("ephemeral wake did not match durable delivery"), + ).await; + continue; + } + let Some((message, owner)) = verified_workflow_delivery_message( + &delivery, + &ctx.rest_client, + &pubkey_hex, + relay_self.as_deref(), + ).await else { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("delivery_verification_failed"), + Some("definition, execution state, or visible message did not verify"), + ).await; + continue; + }; + buzz_event.channel_id = delivery.channel_id; + buzz_event.event = message; + kind_u32 = KIND_STREAM_MESSAGE; + durable_workflow_owner = Some(owner); + claimed_workflow_delivery = Some(delivery); + } if kind_u32 == KIND_MEMBER_ADDED_NOTIFICATION || kind_u32 == KIND_MEMBER_REMOVED_NOTIFICATION @@ -2710,6 +3533,23 @@ async fn tokio_main() -> Result<()> { // complete normally (the relay may reject actions if // the agent lost access). let drained_ids = queue.drain_channel(ch); + for event_id in &drained_ids { + workflow_native_steer_turn_by_event.remove(event_id); + if let Some(delivery) = + workflow_deliveries_by_event.remove(event_id) + { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("channel_membership_removed"), + Some("managed agent lost channel membership"), + ) + .await; + } + } let invalidated = if pool_ready { pool.invalidate_channel_sessions(ch) } else { @@ -2867,28 +3707,32 @@ async fn tokio_main() -> Result<()> { // it never revokes same-owner team bots. { let author = buzz_event.event.pubkey.to_hex(); - // DM hardening: resolve channel type (fail-closed - // to DM) so allowlist/anyone modes cannot be - // exercised by non-owner authors inside DMs. - let is_dm = - is_dm_channel(buzz_event.channel_id, &ctx.channel_info).await; + let doorbell_shape = is_workflow_delivery_candidate( + &buzz_event.event, + relay_self.as_deref(), + ); + // Workflow-shaped visible messages are admitted only after + // a durable kind:24620 wake has been authenticated, claimed, + // and verified against its immutable delivery snapshot. + let Some(principal) = workflow_delivery_principal( + &author, + durable_workflow_owner.as_deref(), + doorbell_shape, + ) else { + tracing::warn!(channel_id = %buzz_event.channel_id, "unclaimed workflow message — dropping fail closed"); + continue; + }; + let is_dm = is_dm_channel(buzz_event.channel_id, &ctx.channel_info).await; let allowed = author_allowed( &config.respond_to, &config.respond_to_allowlist, - &author, + &principal, is_dm, &owner_cache, &ctx.rest_client, - ) - .await; + ).await; if !allowed { - tracing::debug!( - channel_id = %buzz_event.channel_id, - author = %buzz_event.event.pubkey.to_hex(), - mode = %config.respond_to, - is_dm, - "inbound author gate — dropping event" - ); + tracing::debug!(channel_id = %buzz_event.channel_id, author = %author, principal = %principal, workflow_delegated = durable_workflow_owner.is_some(), mode = %config.respond_to, is_dm, "inbound author gate — dropping event"); continue; } } @@ -2929,6 +3773,9 @@ async fn tokio_main() -> Result<()> { // guard's cleanup may race with this add, leaving a // cosmetic stale 👀. Acceptable — see ReactionGuard docs. if accepted { + if let Some(delivery) = claimed_workflow_delivery { + workflow_deliveries_by_event.insert(event_id_hex.clone(), delivery); + } let rc = ctx.rest_client.clone(); let eid = event_id_hex.clone(); tokio::spawn(async move { @@ -2961,16 +3808,25 @@ async fn tokio_main() -> Result<()> { // to the universal cancel+merge `Steer` // signal so the event still reaches the // agent. - let native_attempted = matches!(signal, ControlSignal::Steer) - && try_native_steer( - &mut pool, - &mut queue, - buzz_event.channel_id, - event_for_steer, - prompt_tag_for_steer, - &steer_ack_tx, - ); - if !native_attempted { + let native_turn = matches!(signal, ControlSignal::Steer) + .then(|| { + try_native_steer( + &mut pool, + &mut queue, + buzz_event.channel_id, + event_for_steer, + prompt_tag_for_steer, + &steer_ack_tx, + ) + }) + .flatten(); + if let Some(turn_id) = native_turn.as_ref() { + if workflow_deliveries_by_event.contains_key(&event_id_hex) { + workflow_native_steer_turn_by_event + .insert(event_id_hex.clone(), turn_id.clone()); + } + } + if native_turn.is_none() { signal_in_flight_task( &mut pool, buzz_event.channel_id, @@ -2981,7 +3837,14 @@ async fn tokio_main() -> Result<()> { } if pool_ready { for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) + dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } @@ -3081,7 +3944,14 @@ async fn tokio_main() -> Result<()> { } else if queue.has_flushable_work() { tracing::debug!("heartbeat_skipped_events"); for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) + dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } @@ -3144,6 +4014,19 @@ async fn tokio_main() -> Result<()> { match pool_event { Some(PoolEvent::Result(result)) => { + let turn_id = result.turn_id.clone(); + let workflow_deliveries = workflow_deliveries_by_turn.remove(&turn_id); + let workflow_delivered = matches!(result.outcome, PromptOutcome::Ok(_)); + let workflow_channel = match &result.source { + PromptSource::Channel(channel_id) => Some(*channel_id), + PromptSource::Heartbeat => None, + }; + if workflow_native_steer_turn_by_event + .values() + .any(|pending_turn| pending_turn == &turn_id) + { + completed_workflow_turns.insert(turn_id.clone(), workflow_delivered); + } // Stop typing indicator for the completed channel. if let PromptSource::Channel(ch) = &result.source { typing_channels.remove(ch); @@ -3164,7 +4047,29 @@ async fn tokio_main() -> Result<()> { { break; } - if drain_ready_join_results( + if let Some(deliveries) = workflow_deliveries { + let terminal = retain_workflow_deliveries_for_local_retry( + &queue, + workflow_channel, + !workflow_delivered, + deliveries, + &mut workflow_deliveries_by_event, + ); + for delivery in terminal { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + workflow_delivered, + false, + (!workflow_delivered).then_some("prompt_failed"), + (!workflow_delivered) + .then_some("managed agent turn terminated without a local retry"), + ) + .await; + } + } + let (join_action, panic_recoveries) = drain_ready_join_results( &mut pool, &mut queue, &config, @@ -3175,19 +4080,48 @@ async fn tokio_main() -> Result<()> { &respawn_tx, &mut respawn_tasks, observer.clone(), - ) == LoopAction::Exit - { + ); + for recovery in panic_recoveries { + if let Some(deliveries) = workflow_deliveries_by_turn.remove(&recovery.turn_id) + { + let terminal = retain_workflow_deliveries_for_local_retry( + &queue, + recovery.channel_id, + true, + deliveries, + &mut workflow_deliveries_by_event, + ); + for delivery in terminal { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("prompt_panicked"), + Some("managed agent panicked without a local retry"), + ) + .await; + } + } + } + if join_action == LoopAction::Exit { break; } - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } Some(PoolEvent::Panic(join_error)) => { tracing::error!("agent task panicked: {join_error}"); - recover_panicked_agent( + let panic_recovery = recover_panicked_agent( &mut pool, &mut queue, &config, @@ -3200,19 +4134,49 @@ async fn tokio_main() -> Result<()> { &mut respawn_tasks, observer.clone(), ); + if let Some(recovery) = panic_recovery { + if let Some(deliveries) = workflow_deliveries_by_turn.remove(&recovery.turn_id) + { + let terminal = retain_workflow_deliveries_for_local_retry( + &queue, + recovery.channel_id, + true, + deliveries, + &mut workflow_deliveries_by_event, + ); + for delivery in terminal { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("prompt_panicked"), + Some("managed agent panicked without a local retry"), + ) + .await; + } + } + } if pool.live_count() == 0 && !any_respawn_in_flight(&crash_history) { tracing::error!("all agents dead — exiting"); break; } - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } Some(PoolEvent::SteerAck(SteerAckEvent { channel_id, event_id, + turn_id, ack, })) => { // Mid-turn steer attempt resolved (either transport: @@ -3315,6 +4279,9 @@ async fn tokio_main() -> Result<()> { Ok(pool::SteerAck::PromptCompletedNeutral) => (true, false, false), Err(_recv_err) => (true, false, false), }; + // Every terminal ack resolves this event from the accepting turn. + // Retain the result until all events for that turn resolve. + workflow_native_steer_turn_by_event.remove(&event_id); tracing::info!( channel = %channel_id, event_id = %event_id, @@ -3326,6 +4293,16 @@ async fn tokio_main() -> Result<()> { ); if let Ok(pool::SteerAck::Success { session_id }) = &ack { queue.extend_in_flight_deadline(channel_id, config.max_turn_duration_secs); + transfer_native_steer_workflow_delivery( + &ctx.rest_client, + &event_id, + &turn_id, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + &mut completed_workflow_turns, + &mut pending_workflow_finalizations, + ) + .await; if !pool.record_successful_steer( channel_id, event_id.clone(), @@ -3344,6 +4321,11 @@ async fn tokio_main() -> Result<()> { if release_withheld { queue.release_native_steer(channel_id, &event_id); } + clear_completed_workflow_turn_if_resolved( + &turn_id, + &workflow_native_steer_turn_by_event, + &mut completed_workflow_turns, + ); if signal_fallback { // Universal cancel+merge fallback. Note: the // queued event has already been released to the @@ -3359,9 +4341,14 @@ async fn tokio_main() -> Result<()> { // tear down the in-flight task; on its completion the // queue drains. We still try here in case the in-flight // task has already returned. - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } @@ -3387,9 +4374,14 @@ async fn tokio_main() -> Result<()> { "ready", None, ); - for (channel_id, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (channel_id, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + ) { typing_channels.insert(channel_id, thread_tags); } } @@ -3457,6 +4449,20 @@ async fn tokio_main() -> Result<()> { maybe_result = rx_ref.recv() => { if let Some(mut pr) = maybe_result { let idx = pr.agent.index; + let delivered = matches!(pr.outcome, PromptOutcome::Ok(_)); + if let Some(deliveries) = workflow_deliveries_by_turn.remove(&pr.turn_id) { + for delivery in deliveries { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + delivered, + false, + (!delivered).then_some("runtime_shutdown"), + (!delivered).then_some("managed agent stopped during the turn"), + ).await; + } + } pr.agent.acp.shutdown().await; tracing::debug!(agent = idx, "reaped checked-out agent on shutdown"); } @@ -3474,9 +4480,86 @@ async fn tokio_main() -> Result<()> { // before tasks were aborted. while let Ok(mut pr) = pool.result_rx_try_recv() { let idx = pr.agent.index; + let delivered = matches!(pr.outcome, PromptOutcome::Ok(_)); + if let Some(deliveries) = workflow_deliveries_by_turn.remove(&pr.turn_id) { + for delivery in deliveries { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + delivered, + false, + (!delivered).then_some("runtime_shutdown"), + (!delivered).then_some("managed agent stopped during the turn"), + ) + .await; + } + } pr.agent.acp.shutdown().await; tracing::debug!(agent = idx, "reaped late-arriving agent on shutdown"); } + + // Anything still owned after the bounded drain is uncertain or never + // started. Fence it terminally while this runtime still owns the lease; + // never let shutdown silently drop a claim ledger for later redelivery. + for (_, delivery) in workflow_deliveries_by_event.drain() { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("runtime_shutdown"), + Some("managed agent stopped before delivery"), + ) + .await; + } + for delivery in workflow_deliveries_by_turn + .drain() + .flat_map(|(_, deliveries)| deliveries) + { + finish_workflow_delivery( + &ctx.rest_client, + &mut pending_workflow_finalizations, + &delivery, + false, + false, + Some("runtime_shutdown"), + Some("managed agent turn did not finish before shutdown"), + ) + .await; + } + // Renew before every admitted, concurrent finish round so stalled I/O cannot + // starve another claim's lease. The 30-second deadline below stops new rounds; + // an already-admitted renewal/finalization round may finish after it. If + // shutdown still ends with unresolved + // intent, claim expiry is terminal in the database and cannot be reclaimed. + let finalization_deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while !pending_workflow_finalizations.is_empty() + && tokio::time::Instant::now() < finalization_deadline + { + renew_owned_workflow_deliveries( + &ctx.rest_client, + workflow_lease_seconds(&config), + &mut pool, + &mut queue, + &mut workflow_deliveries_by_event, + &mut workflow_deliveries_by_turn, + &mut workflow_native_steer_turn_by_event, + &mut completed_workflow_turns, + &mut pending_workflow_finalizations, + ) + .await; + reconcile_workflow_finalizations(&ctx.rest_client, &mut pending_workflow_finalizations) + .await; + tokio::time::sleep(Duration::from_secs(1)).await; + } + if !pending_workflow_finalizations.is_empty() { + tracing::error!( + deliveries = pending_workflow_finalizations.len(), + "shutdown ended with unresolved workflow finalizations; claim expiry will terminalize them without redelivery" + ); + } // Explicitly shut down idle agents still sitting in their slots. for slot in pool.agents_mut().iter_mut() { if let Some(agent) = slot.take() { @@ -3639,7 +4722,7 @@ fn try_native_steer( event: nostr::Event, prompt_tag: String, steer_ack_tx: &mpsc::UnboundedSender, -) -> bool { +) -> Option { // Build the steer body: framing strings come from // `queue::native_steer_framing()` (Eva's drift-proof requirement — // native and cancel+merge fallback share these so the agent gets the @@ -3670,7 +4753,7 @@ fn try_native_steer( }; match pool.send_steer(channel_id, request) { - Ok(()) => { + Ok(turn_id) => { // Withhold the queued event synchronously BEFORE spawning // the watcher: this closes the race where `mark_complete` // clears `in_flight_channels` and a stray `flush_next` could @@ -3694,15 +4777,17 @@ fn try_native_steer( } let ack_tx_clone = steer_ack_tx.clone(); let event_id_for_watcher = event_id_hex.clone(); + let ack_turn_id = turn_id.clone(); tokio::spawn(async move { let ack = ack_rx.await; let _ = ack_tx_clone.send(SteerAckEvent { channel_id, event_id: event_id_for_watcher, + turn_id: ack_turn_id, ack, }); }); - true + Some(turn_id) } Err(e) => { tracing::info!( @@ -3710,19 +4795,32 @@ fn try_native_steer( error = ?e, "non-cancelling steer not accepted — falling back to cancel+merge" ); - false + None } } } // ── dispatch_pending ────────────────────────────────────────────────────────── +/// Retain a checked-out batch for failure recovery when Queue semantics +/// require it, or when a claimed durable workflow delivery needs exact batch +/// ownership independent of the global deduplication mode. +fn recoverable_dispatch_batch( + dedup_mode: DedupMode, + has_workflow_delivery: bool, + batch: &FlushBatch, +) -> Option { + (matches!(dedup_mode, DedupMode::Queue) || has_workflow_delivery).then(|| batch.clone()) +} + /// Flush queued work to available agents. fn dispatch_pending( pool: &mut AgentPool, queue: &mut EventQueue, ctx: &Arc, last_activity: &mut tokio::time::Instant, + workflow_deliveries_by_event: &mut HashMap, + workflow_deliveries_by_turn: &mut HashMap>, ) -> Vec<(Uuid, ThreadTags)> { let mut dispatched_channels = Vec::new(); loop { @@ -3749,11 +4847,6 @@ fn dispatch_pending( }; tracing::debug!(agent = agent.index, channel = %channel_id, affinity_hit, "agent_claimed"); - let recoverable_batch = match ctx.dedup_mode { - DedupMode::Queue => Some(batch.clone()), - DedupMode::Drop => None, - }; - let result_tx = pool.result_tx(); let ctx_clone = Arc::clone(ctx); let agent_index = agent.index; @@ -3776,6 +4869,23 @@ fn dispatch_pending( let (control_tx, control_rx) = tokio::sync::oneshot::channel::(); let turn_id = Uuid::new_v4().to_string(); let task_turn_id = turn_id.clone(); + let task_workflow_deliveries = batch + .events + .iter() + .chain(batch.cancelled_events.iter()) + .filter_map(|event| workflow_deliveries_by_event.remove(&event.event.id.to_hex())) + .collect::>(); + let has_workflow_delivery = !task_workflow_deliveries.is_empty(); + if has_workflow_delivery { + workflow_deliveries_by_turn.insert(turn_id.clone(), task_workflow_deliveries); + } + // Queue mode recovers every failed turn. Drop mode keeps its ordinary + // fire-and-forget semantics, except for batches carrying a claimed + // durable workflow delivery: terminal lease handling and panic + // recovery must still own the exact checked-out batch long enough to + // remove the terminal workflow event without losing co-batched work. + let recoverable_batch = + recoverable_dispatch_batch(ctx.dedup_mode, has_workflow_delivery, &batch); let abort_handle = pool.join_set.spawn(async move { pool::run_prompt_task( @@ -4254,6 +5364,12 @@ fn handle_prompt_result( LoopAction::Continue } +#[derive(Debug)] +struct PanicRecovery { + turn_id: String, + channel_id: Option, +} + #[allow(clippy::too_many_arguments)] fn recover_panicked_agent( pool: &mut AgentPool, @@ -4267,11 +5383,15 @@ fn recover_panicked_agent( respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, -) { +) -> Option { let task_id = join_error.id(); let Some(meta) = pool.task_map_mut().remove(&task_id) else { tracing::error!("panic for unknown task {task_id:?} — bug"); - return; + return None; + }; + let recovery = PanicRecovery { + turn_id: meta.turn_id.clone(), + channel_id: meta.channel_id, }; let i = meta.agent_index; @@ -4321,7 +5441,7 @@ fn recover_panicked_agent( let delay = match slot.record_crash() { CrashVerdict::CircuitOpen => { tracing::error!(agent = i, "circuit open after panic — not respawning"); - return; + return Some(recovery); } CrashVerdict::HalfOpenProbe => { tracing::info!(agent = i, "circuit half-open — probe respawn after panic"); @@ -4351,6 +5471,7 @@ fn recover_panicked_agent( let result = spawn_and_init(&cmd, &args, &env, has_codex, i, observer).await; guard.send(result); }); + Some(recovery) } #[allow(clippy::too_many_arguments)] @@ -4365,11 +5486,12 @@ fn drain_ready_join_results( respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, -) -> LoopAction { +) -> (LoopAction, Vec) { + let mut recoveries = Vec::new(); while let Some(Some(join_result)) = pool.join_set.join_next().now_or_never() { if let Err(join_error) = join_result { tracing::error!("agent task panicked: {join_error}"); - recover_panicked_agent( + if let Some(recovery) = recover_panicked_agent( pool, queue, config, @@ -4381,13 +5503,15 @@ fn drain_ready_join_results( respawn_tx, respawn_tasks, observer.clone(), - ); + ) { + recoveries.push(recovery); + } if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { - return LoopAction::Exit; + return (LoopAction::Exit, recoveries); } } } - LoopAction::Continue + (LoopAction::Continue, recoveries) } fn dispatch_heartbeat( @@ -5087,6 +6211,828 @@ fn build_mcp_servers(config: &Config) -> Vec { }] } +#[cfg(test)] +mod workflow_authority_tests { + use super::*; + use nostr::{EventBuilder, Keys, Kind, Tag}; + + #[allow(clippy::too_many_arguments)] + fn workflow_wake( + signer: &Keys, + target: &Keys, + channel: Uuid, + delivery_id: Uuid, + definition_id: &str, + run_id: Uuid, + step_id: &str, + message_id: &str, + extra_tags: impl IntoIterator, + ) -> nostr::Event { + let mut tags = vec![ + Tag::parse(["p", &target.public_key().to_hex()]).unwrap(), + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["delivery", &delivery_id.to_string()]).unwrap(), + Tag::parse(["workflow-definition", definition_id]).unwrap(), + Tag::parse(["workflow-run", &run_id.to_string()]).unwrap(), + Tag::parse(["workflow-step", step_id]).unwrap(), + Tag::parse(["message", message_id]).unwrap(), + ]; + tags.extend(extra_tags); + EventBuilder::new(Kind::Custom(KIND_WORKFLOW_AGENT_WAKE as u16), "") + .tags(tags) + .sign_with_keys(signer) + .unwrap() + } + + #[test] + fn live_workflow_wake_authentication_is_exact_and_channel_bound() { + let relay = Keys::generate(); + let attacker = Keys::generate(); + let agent = Keys::generate(); + let channel = Uuid::new_v4(); + let other_channel = Uuid::new_v4(); + let delivery_id = Uuid::new_v4(); + let run_id = Uuid::new_v4(); + let definition_id = "11".repeat(32); + let message_id = "22".repeat(32); + let valid = workflow_wake( + &relay, + &agent, + channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [], + ); + let authenticate = |event: &nostr::Event, receiving_channel| { + trusted_live_workflow_wake_delivery_id( + event, + receiving_channel, + &agent.public_key().to_hex(), + Some(&relay.public_key().to_hex()), + ) + }; + assert_eq!( + authenticate(&valid, channel).map(|(id, _)| id), + Some(delivery_id) + ); + assert_eq!(authenticate(&valid, other_channel), None); + + let forged_author = workflow_wake( + &attacker, + &agent, + channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [], + ); + assert_eq!(authenticate(&forged_author, channel), None); + + let mut invalid_signature = valid.clone(); + invalid_signature.content = "tampered".into(); + assert_eq!(authenticate(&invalid_signature, channel), None); + + let duplicate_delivery = workflow_wake( + &relay, + &agent, + channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [Tag::parse(["delivery", &Uuid::new_v4().to_string()]).unwrap()], + ); + assert_eq!(authenticate(&duplicate_delivery, channel), None); + + for malformed in [ + workflow_wake( + &relay, + &agent, + channel, + delivery_id, + "not-an-event-id", + run_id, + "wake", + &message_id, + [], + ), + workflow_wake( + &relay, + &agent, + channel, + delivery_id, + &definition_id, + run_id, + "", + &message_id, + [], + ), + ] { + assert_eq!(authenticate(&malformed, channel), None); + } + } + + #[tokio::test] + async fn untrusted_two_channel_live_wakes_never_reach_claim_or_finish() { + use tokio::io::AsyncReadExt; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let observed = tokio::spawn(async move { + match tokio::time::timeout(Duration::from_millis(300), listener.accept()).await { + Ok(Ok((mut socket, _))) => { + let mut request = vec![0; 8192]; + let read = socket.read(&mut request).await.unwrap(); + Some(String::from_utf8_lossy(&request[..read]).into_owned()) + } + _ => None, + } + }); + let client = relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }; + let relay = Keys::generate(); + let attacker = Keys::generate(); + let agent = Keys::generate(); + let victim_channel = Uuid::new_v4(); + let attacker_channel = Uuid::new_v4(); + let delivery_id = Uuid::new_v4(); + let run_id = Uuid::new_v4(); + let definition_id = "11".repeat(32); + let message_id = "22".repeat(32); + let forged = workflow_wake( + &attacker, + &agent, + victim_channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [], + ); + let duplicate = workflow_wake( + &relay, + &agent, + victim_channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [Tag::parse(["workflow-run", &Uuid::new_v4().to_string()]).unwrap()], + ); + let wrong_channel = workflow_wake( + &relay, + &agent, + victim_channel, + delivery_id, + &definition_id, + run_id, + "wake", + &message_id, + [], + ); + + for (wake, receiving_channel) in [ + (&forged, victim_channel), + (&duplicate, victim_channel), + (&wrong_channel, attacker_channel), + ] { + if let Some((id, binding)) = trusted_live_workflow_wake_delivery_id( + wake, + receiving_channel, + &agent.public_key().to_hex(), + Some(&relay.public_key().to_hex()), + ) { + let claimed = claim_workflow_delivery(&client, Some(id), Some(&binding), 120).await; + if let Some(delivery) = claimed { + let mut finalizations = HashMap::new(); + finish_workflow_delivery( + &client, + &mut finalizations, + &delivery, + false, + false, + Some("binding_mismatch"), + Some("invalid wake"), + ) + .await; + reconcile_workflow_finalizations(&client, &mut finalizations).await; + } + } + } + assert_eq!( + observed.await.unwrap(), + None, + "forged, duplicate, and cross-channel wakes must invoke neither claim nor finish" + ); + } + + #[tokio::test] + async fn renewal_failure_records_one_terminal_finalization_intent() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 8192]; + let read = socket.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = "{}"; + let response = format!("HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + socket.write_all(response.as_bytes()).await.unwrap(); + request + }); + let client = relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }; + let channel = Uuid::new_v4(); + let event = EventBuilder::new(Kind::Custom(KIND_STREAM_MESSAGE as u16), "workflow") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let event_id = event.id.to_hex(); + let delivery = ClaimedWorkflowDelivery { + id: Uuid::new_v4(), + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "step".into(), + definition_event_id: "11".repeat(32), + message_event_id: event_id.clone(), + channel_id: channel, + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now(), + execution_trace: serde_json::json!([]), + trigger_context: Some(serde_json::json!({})), + }; + let delivery_id = delivery.id; + let mut queue = EventQueue::new(config::DedupMode::Queue); + assert!(queue.push(QueuedEvent { + channel_id: channel, + event, + received_at: std::time::Instant::now(), + prompt_tag: "workflow".into() + })); + let _checked_out = queue.flush_next().expect("workflow batch in flight"); + let mut pool = AgentPool::from_slots(vec![]); + let mut by_event = HashMap::from([(event_id, delivery)]); + let mut by_turn = HashMap::new(); + let mut native = HashMap::new(); + let mut completed = HashMap::new(); + let mut finalizing = HashMap::new(); + renew_owned_workflow_deliveries( + &client, + 120, + &mut pool, + &mut queue, + &mut by_event, + &mut by_turn, + &mut native, + &mut completed, + &mut finalizing, + ) + .await; + let request = server.await.unwrap(); + assert!(request.contains(&format!("/workflows/agent-deliveries/{delivery_id}/renew"))); + assert!(by_event.is_empty()); + assert_eq!(finalizing.len(), 1, "one terminal intent per delivery"); + let terminal = finalizing.get(&delivery_id).expect("terminal intent"); + assert!(!terminal.delivered); + assert!(!terminal.retryable); + assert_eq!( + terminal.failure_code.as_deref(), + Some("lease_renewal_failed") + ); + renew_owned_workflow_deliveries( + &client, + 120, + &mut pool, + &mut queue, + &mut by_event, + &mut by_turn, + &mut native, + &mut completed, + &mut finalizing, + ) + .await; + assert_eq!( + finalizing.len(), + 1, + "renewal handling cannot duplicate finalization" + ); + } + + #[test] + fn maximum_supported_turn_requests_an_accepted_workflow_lease() { + let mut config = crate::config::Config::from_args(crate::config::CliArgs::parse_from([ + "buzz-acp", + "--private-key", + "0000000000000000000000000000000000000000000000000000000000000001", + ])) + .expect("test config"); + config.max_turn_duration_secs = config::MAX_TURN_DURATION_CEILING_SECS; + assert_eq!( + workflow_lease_seconds(&config), + buzz_core::workflow_delivery::MAX_LEASE_SECONDS + ); + } + + #[tokio::test] + async fn stalled_finalizations_share_one_safe_concurrent_budget() { + use tokio::io::AsyncReadExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let mut sockets = Vec::new(); + for _ in 0..2 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + sockets.push(socket); + } + tokio::time::sleep(Duration::from_secs(3)).await; + sockets.len() + }); + let client = relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }; + let claim_expires_at = chrono::Utc::now() + + chrono::Duration::seconds(WORKFLOW_LEASE_FAIL_CLOSED_MARGIN_SECS + 5); + let mut pending = HashMap::new(); + for marker in ["aa", "bb"] { + let delivery = ClaimedWorkflowDelivery { + id: Uuid::new_v4(), + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "step".into(), + definition_event_id: "11".repeat(32), + message_event_id: marker.repeat(32), + channel_id: Uuid::new_v4(), + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at, + execution_trace: serde_json::json!([]), + trigger_context: Some(serde_json::json!({})), + }; + finish_workflow_delivery(&client, &mut pending, &delivery, true, false, None, None) + .await; + } + let started = tokio::time::Instant::now(); + reconcile_workflow_finalizations(&client, &mut pending).await; + assert!(started.elapsed() < Duration::from_secs(6)); + assert_eq!( + pending.len(), + 2, + "stalled finishes retain exact terminal intent" + ); + assert_eq!( + server.await.unwrap(), + 2, + "both finalizations start concurrently" + ); + } + + async fn rest_client_serving( + responses: Vec, + ) -> (relay::RestClient, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + for response_body in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 8192]; + let _ = socket.read(&mut request).await; + let body = response_body.to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), body + ); + socket.write_all(response.as_bytes()).await.unwrap(); + } + }); + ( + relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }, + server, + ) + } + + #[test] + fn only_verified_durable_workflow_messages_reach_dispatch() { + assert_eq!( + workflow_delivery_principal("relay", Some("durable-owner"), true), + Some("durable-owner".to_owned()) + ); + assert!( + workflow_delivery_principal("relay", None, true).is_none(), + "an unclaimed workflow-shaped message must fail closed" + ); + assert_eq!( + workflow_delivery_principal("human", None, false), + Some("human".to_owned()), + "ordinary messages retain their author principal" + ); + } + + #[tokio::test] + async fn claimed_native_steer_finishes_once_with_exact_completed_turn() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16384]; + let read = socket.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = "{}"; + let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + socket.write_all(response.as_bytes()).await.unwrap(); + request + }); + let client = relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }; + let channel = Uuid::new_v4(); + let event = EventBuilder::new(Kind::Custom(KIND_STREAM_MESSAGE as u16), "durable steer") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let event_id = event.id.to_hex(); + let delivery_id = Uuid::new_v4(); + let mut queue = EventQueue::new(config::DedupMode::Queue); + assert!(queue.push(QueuedEvent { + channel_id: channel, + event, + received_at: std::time::Instant::now(), + prompt_tag: "workflow".into() + })); + assert!(queue.mark_native_steer_pending(channel, &event_id)); + let mut by_event = HashMap::from([( + event_id.clone(), + ClaimedWorkflowDelivery { + id: delivery_id, + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "step".into(), + definition_event_id: "11".repeat(32), + message_event_id: event_id.clone(), + channel_id: channel, + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + execution_trace: serde_json::json!([]), + trigger_context: Some(serde_json::json!({})), + }, + )]); + let mut by_turn = HashMap::new(); + let mut completed = HashMap::from([("busy-turn".into(), true)]); + let mut finalizing = HashMap::new(); + let mut pending_turn_by_event = HashMap::from([(event_id.clone(), "busy-turn".to_owned())]); + transfer_native_steer_workflow_delivery( + &client, + &event_id, + "busy-turn", + &mut by_event, + &mut by_turn, + &mut completed, + &mut finalizing, + ) + .await; + assert!(by_event.is_empty() && by_turn.is_empty()); + assert_eq!(completed.get("busy-turn"), Some(&true)); + pending_turn_by_event.remove(&event_id); + clear_completed_workflow_turn_if_resolved( + "busy-turn", + &pending_turn_by_event, + &mut completed, + ); + queue.remove_event(channel, &event_id); + assert!(completed.is_empty()); + assert_eq!(queue.queued_event_count(&channel), 0); + reconcile_workflow_finalizations(&client, &mut finalizing).await; + let request = server.await.unwrap(); + assert!(request.contains(&format!("/workflows/agent-deliveries/{delivery_id}/finish"))); + assert!(request.contains("\"delivered\":true")); + // Idempotent ack replay cannot finish the removed fenced claim twice. + transfer_native_steer_workflow_delivery( + &client, + &event_id, + "busy-turn", + &mut by_event, + &mut by_turn, + &mut completed, + &mut finalizing, + ) + .await; + } + + #[tokio::test] + async fn two_successful_native_steers_finish_against_one_prior_turn_result() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let mut requests = Vec::new(); + for _ in 0..2 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16384]; + let read = socket.read(&mut request).await.unwrap(); + requests.push(String::from_utf8_lossy(&request[..read]).into_owned()); + let body = "{}"; + let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + socket.write_all(response.as_bytes()).await.unwrap(); + } + requests + }); + let client = relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys: Keys::generate(), + auth_tag_json: None, + }; + let turn_id = "shared-completed-turn"; + let event_ids = ["aa".repeat(32), "bb".repeat(32)]; + let delivery_ids = [Uuid::new_v4(), Uuid::new_v4()]; + let mut by_event = HashMap::new(); + for (event_id, delivery_id) in event_ids.iter().zip(delivery_ids) { + by_event.insert( + event_id.clone(), + ClaimedWorkflowDelivery { + id: delivery_id, + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "step".into(), + definition_event_id: "11".repeat(32), + message_event_id: event_id.clone(), + channel_id: Uuid::new_v4(), + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + execution_trace: serde_json::json!([]), + trigger_context: Some(serde_json::json!({})), + }, + ); + } + let mut pending = HashMap::from([ + (event_ids[0].clone(), turn_id.into()), + (event_ids[1].clone(), turn_id.into()), + ]); + let mut by_turn = HashMap::new(); + let mut completed = HashMap::from([(turn_id.into(), true)]); + let mut finalizing = HashMap::new(); + for event_id in &event_ids { + pending.remove(event_id); + transfer_native_steer_workflow_delivery( + &client, + event_id, + turn_id, + &mut by_event, + &mut by_turn, + &mut completed, + &mut finalizing, + ) + .await; + clear_completed_workflow_turn_if_resolved(turn_id, &pending, &mut completed); + } + assert!(by_event.is_empty() && by_turn.is_empty() && completed.is_empty()); + reconcile_workflow_finalizations(&client, &mut finalizing).await; + let requests = server.await.unwrap(); + assert_eq!(requests.len(), 2); + for delivery_id in delivery_ids { + assert!(requests.iter().any(|request| request + .contains(&format!("/workflows/agent-deliveries/{delivery_id}/finish")))); + } + assert!(requests + .iter() + .all(|request| request.contains("\"delivered\":true"))); + } + + #[test] + fn mixed_native_steer_resolution_clears_outcome_only_after_final_ack() { + let turn_id = "mixed-turn"; + let mut pending = HashMap::from([ + ("success".to_string(), turn_id.into()), + ("failed".to_string(), turn_id.into()), + ]); + let mut completed = HashMap::from([(turn_id.into(), true)]); + pending.remove("failed"); + clear_completed_workflow_turn_if_resolved(turn_id, &pending, &mut completed); + assert_eq!(completed.get(turn_id), Some(&true)); + pending.remove("success"); + clear_completed_workflow_turn_if_resolved(turn_id, &pending, &mut completed); + assert!(!completed.contains_key(turn_id)); + } + + #[test] + fn rejected_native_steer_retains_claim_on_released_queue_event() { + let channel = Uuid::new_v4(); + let event = EventBuilder::new(Kind::Custom(KIND_STREAM_MESSAGE as u16), "durable fallback") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let event_id = event.id.to_hex(); + let mut queue = EventQueue::new(config::DedupMode::Queue); + assert!(queue.push(QueuedEvent { + channel_id: channel, + event, + received_at: std::time::Instant::now(), + prompt_tag: "workflow".into() + })); + assert!(queue.mark_native_steer_pending(channel, &event_id)); + let claim = ClaimedWorkflowDelivery { + id: Uuid::new_v4(), + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "step".into(), + definition_event_id: "11".repeat(32), + message_event_id: event_id.clone(), + channel_id: channel, + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + execution_trace: serde_json::json!([]), + trigger_context: Some(serde_json::json!({})), + }; + let mut by_event = HashMap::from([(event_id.clone(), claim)]); + queue.release_native_steer(channel, &event_id); + let batch = queue + .flush_next() + .expect("released fallback dispatches normally"); + let turn_claims = batch + .events + .iter() + .filter_map(|event| by_event.remove(&event.event.id.to_hex())) + .collect::>(); + assert_eq!(turn_claims.len(), 1); + assert!(by_event.is_empty()); + } + + #[tokio::test] + async fn durable_claim_reconstructs_prior_output_and_rejects_mutable_bindings() { + let relay = Keys::generate(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let channel = Uuid::new_v4(); + let workflow_id = Uuid::new_v4(); + let run_id = Uuid::new_v4(); + let definition = EventBuilder::new( + Kind::Custom(buzz_core::kind::KIND_WORKFLOW_DEF as u16), + "name: signed\ntrigger:\n on: webhook\nsteps:\n - id: call\n action: send_message\n text: prior\n - id: wake\n action: send_message\n text: 'status {{steps.call.output.body}}'\n", + ) + .tags([ + Tag::parse(["d", &workflow_id.to_string()]).unwrap(), + Tag::parse(["h", &channel.to_string()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let message = EventBuilder::new(Kind::Custom(9), "status durable") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["p", &agent.public_key().to_hex()]).unwrap(), + Tag::parse(["workflow-definition", &definition.id.to_hex()]).unwrap(), + Tag::parse(["workflow-run", &run_id.to_string()]).unwrap(), + Tag::parse(["workflow-step", "wake"]).unwrap(), + ]) + .sign_with_keys(&relay) + .unwrap(); + let trigger = buzz_workflow::executor::TriggerContext { + channel_id: channel.to_string(), + definition_event_id: definition.id.to_hex(), + cause: Some(buzz_workflow::executor::WorkflowCause::Webhook), + ..Default::default() + }; + let delivery = ClaimedWorkflowDelivery { + id: Uuid::new_v4(), + workflow_id, + run_id, + step_id: "wake".to_string(), + definition_event_id: definition.id.to_hex(), + message_event_id: message.id.to_hex(), + channel_id: channel, + target_pubkey: agent.public_key().to_hex(), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + execution_trace: serde_json::json!([{ + "step_id": "call", + "output": {"body": "durable"} + }]), + trigger_context: Some(serde_json::to_value(trigger).unwrap()), + }; + let mut candidates = vec![delivery.clone()]; + candidates[0].workflow_id = Uuid::new_v4(); + candidates.push({ + let mut d = delivery.clone(); + d.run_id = Uuid::new_v4(); + d + }); + candidates.push({ + let mut d = delivery.clone(); + d.step_id = "call".into(); + d + }); + candidates.push({ + let mut d = delivery.clone(); + d.channel_id = Uuid::new_v4(); + d + }); + candidates.push({ + let mut d = delivery.clone(); + d.target_pubkey = owner.public_key().to_hex(); + d + }); + candidates.push({ + let mut d = delivery.clone(); + d.definition_event_id = message.id.to_hex(); + d + }); + candidates.push({ + let mut d = delivery.clone(); + d.message_event_id = definition.id.to_hex(); + d + }); + + let (client, server) = rest_client_serving(vec![ + serde_json::json!([definition.clone()]), + serde_json::json!([message.clone()]), + ]) + .await; + let verified = verified_workflow_delivery_message( + &delivery, + &client, + &agent.public_key().to_hex(), + Some(&relay.public_key().to_hex()), + ) + .await + .expect("durable trace reconstructs prior-step template"); + assert_eq!(verified.0.content, "status durable"); + assert_eq!(verified.1, owner.public_key().to_hex()); + server.await.unwrap(); + + for mutated in candidates { + let (client, server) = rest_client_serving(vec![ + serde_json::json!([definition.clone()]), + serde_json::json!([message.clone()]), + ]) + .await; + assert!(verified_workflow_delivery_message( + &mutated, + &client, + &agent.public_key().to_hex(), + Some(&relay.public_key().to_hex()), + ) + .await + .is_none()); + server.abort(); + } + } + + #[test] + fn unclaimed_workflow_messages_fail_closed_regardless_of_p_tags() { + let relay = Keys::generate(); + let owner = Keys::generate().public_key().to_hex(); + let event = EventBuilder::new(Kind::Custom(KIND_STREAM_MESSAGE as u16), "visible") + .tags([ + Tag::parse(["buzz:workflow", "message-v1"]).unwrap(), + Tag::parse(["p", &owner]).unwrap(), + ]) + .sign_with_keys(&relay) + .unwrap(); + assert!(is_workflow_delivery_candidate( + &event, + Some(&relay.public_key().to_hex()) + )); + assert!(workflow_delivery_principal("relay", None, true).is_none()); + } +} + #[cfg(test)] mod heartbeat_base_prompt_tests { use super::*; @@ -8461,6 +10407,352 @@ mod error_outcome_emission_tests { "non-auth application error must preserve the event for retry" ); } + + #[test] + fn drop_mode_retains_only_workflow_batches_for_failure_recovery() { + let channel_id = Uuid::new_v4(); + let keys = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(9), "test") + .sign_with_keys(&keys) + .unwrap(); + let batch = FlushBatch { + channel_id, + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + }; + + assert!(recoverable_dispatch_batch(DedupMode::Queue, false, &batch).is_some()); + assert!(recoverable_dispatch_batch(DedupMode::Drop, true, &batch).is_some()); + assert!( + recoverable_dispatch_batch(DedupMode::Drop, false, &batch).is_none(), + "ordinary Drop-mode traffic must not become retryable" + ); + } + + #[tokio::test] + async fn terminal_workflow_non_cancel_error_preserves_only_unrelated_events() { + let keys = Keys::generate(); + let workflow = EventBuilder::new(Kind::Custom(9), "terminal workflow") + .sign_with_keys(&keys) + .unwrap(); + let unrelated = EventBuilder::new(Kind::Custom(9), "unrelated") + .sign_with_keys(&keys) + .unwrap(); + let channel_id = Uuid::new_v4(); + let mut queue = EventQueue::new(config::DedupMode::Queue); + for event in [workflow.clone(), unrelated.clone()] { + queue.push(QueuedEvent { + channel_id, + event, + received_at: std::time::Instant::now(), + prompt_tag: "test".into(), + }); + } + let batch = queue.flush_next().expect("production checked-out batch"); + queue.terminalize_workflow_event(channel_id, &workflow.id.to_hex()); + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + turn_id: "terminal-error".into(), + recoverable_batch: Some(batch.clone()), + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + let mut heartbeat_in_flight = false; + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + handle_prompt_result( + &mut pool, + &mut queue, + &test_config(), + PromptResult { + agent, + source: PromptSource::Channel(channel_id), + turn_id: "terminal-error".into(), + outcome: PromptOutcome::Error(AcpError::AgentError { + code: -32000, + message: "retryable".into(), + }), + batch: Some(batch), + }, + &mut heartbeat_in_flight, + &HashSet::new(), + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + ); + assert!(!queue.contains_event(channel_id, &workflow.id.to_hex())); + assert!(queue.contains_event(channel_id, &unrelated.id.to_hex())); + assert_eq!(queue.queued_event_count(&channel_id), 1); + } + + #[tokio::test] + async fn terminal_workflow_panic_recovery_preserves_only_unrelated_events() { + let keys = Keys::generate(); + let workflow = EventBuilder::new(Kind::Custom(9), "terminal workflow") + .sign_with_keys(&keys) + .unwrap(); + let unrelated = EventBuilder::new(Kind::Custom(9), "unrelated") + .sign_with_keys(&keys) + .unwrap(); + let channel_id = Uuid::new_v4(); + let mut queue = EventQueue::new(config::DedupMode::Queue); + for event in [workflow.clone(), unrelated.clone()] { + queue.push(QueuedEvent { + channel_id, + event, + received_at: std::time::Instant::now(), + prompt_tag: "test".into(), + }); + } + let batch = queue.flush_next().expect("production checked-out batch"); + queue.terminalize_workflow_event(channel_id, &workflow.id.to_hex()); + let mut pool = AgentPool::from_slots(vec![]); + let abort_handle = pool.join_set.spawn(std::future::pending::<()>()); + pool.task_map_mut().insert( + abort_handle.id(), + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + turn_id: "terminal-panic".into(), + recoverable_batch: Some(batch), + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + abort_handle.abort(); + let join_error = pool.join_set.join_next().await.unwrap().unwrap_err(); + let mut heartbeat_in_flight = false; + let mut typing_channels = HashMap::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + recover_panicked_agent( + &mut pool, + &mut queue, + &test_config(), + join_error, + &mut heartbeat_in_flight, + &HashSet::new(), + &mut typing_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + ) + .expect("panic metadata survives recovery"); + assert!(!queue.contains_event(channel_id, &workflow.id.to_hex())); + assert!(queue.contains_event(channel_id, &unrelated.id.to_hex())); + assert_eq!(queue.queued_event_count(&channel_id), 1); + } + + fn claimed_delivery(event: &nostr::Event, channel_id: Uuid) -> ClaimedWorkflowDelivery { + ClaimedWorkflowDelivery { + id: Uuid::new_v4(), + workflow_id: Uuid::new_v4(), + run_id: Uuid::new_v4(), + step_id: "deliver".into(), + definition_event_id: "11".repeat(32), + message_event_id: event.id.to_hex(), + channel_id, + target_pubkey: "22".repeat(32), + attempt: 1, + claim_token: Uuid::new_v4(), + claim_expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + execution_trace: serde_json::json!([]), + trigger_context: None, + } + } + + #[tokio::test] + async fn failed_requeued_turn_retains_same_durable_claim_locally() { + let keys = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(9), "durable retry") + .sign_with_keys(&keys) + .unwrap(); + let channel_id = Uuid::new_v4(); + let batch = FlushBatch { + channel_id, + events: vec![BatchEvent { + event: event.clone(), + prompt_tag: "workflow".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + }; + let delivery = claimed_delivery(&event, channel_id); + let original_token = delivery.claim_token; + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + turn_id: "durable-failure-turn".into(), + recoverable_batch: Some(batch.clone()), + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + let mut queue = EventQueue::new(config::DedupMode::Queue); + let mut heartbeat_in_flight = false; + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + handle_prompt_result( + &mut pool, + &mut queue, + &test_config(), + PromptResult { + agent, + source: PromptSource::Channel(channel_id), + turn_id: "durable-failure-turn".into(), + outcome: PromptOutcome::Error(AcpError::AgentError { + code: -32000, + message: "retryable".into(), + }), + batch: Some(batch), + }, + &mut heartbeat_in_flight, + &HashSet::new(), + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + ); + + let mut by_event = HashMap::new(); + let terminal = retain_workflow_deliveries_for_local_retry( + &queue, + Some(channel_id), + true, + vec![delivery], + &mut by_event, + ); + assert!( + terminal.is_empty(), + "local retry must not terminalize the DB claim" + ); + assert_eq!(by_event[&event.id.to_hex()].claim_token, original_token); + assert!(queue.contains_event(channel_id, &event.id.to_hex())); + } + + #[tokio::test] + async fn panic_recovery_returns_same_durable_claim_to_exact_queued_event() { + let keys = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(9), "durable panic retry") + .sign_with_keys(&keys) + .unwrap(); + let channel_id = Uuid::new_v4(); + let batch = FlushBatch { + channel_id, + events: vec![BatchEvent { + event: event.clone(), + prompt_tag: "workflow".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + }; + let delivery = claimed_delivery(&event, channel_id); + let original_token = delivery.claim_token; + let mut pool = AgentPool::from_slots(vec![]); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let abort_handle = pool.join_set.spawn(async move { + let _ = started_tx.send(()); + std::future::pending::<()>().await; + }); + pool.task_map_mut().insert( + abort_handle.id(), + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + turn_id: "durable-panic-turn".into(), + recoverable_batch: Some(batch), + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + started_rx.await.unwrap(); + abort_handle.abort(); + let join_error = pool.join_set.join_next().await.unwrap().unwrap_err(); + let mut queue = EventQueue::new(config::DedupMode::Queue); + let mut heartbeat_in_flight = false; + let mut typing_channels = HashMap::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + let recovery = recover_panicked_agent( + &mut pool, + &mut queue, + &test_config(), + join_error, + &mut heartbeat_in_flight, + &HashSet::new(), + &mut typing_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + ) + .expect("panic metadata survives recovery"); + + let mut by_event = HashMap::new(); + let terminal = retain_workflow_deliveries_for_local_retry( + &queue, + recovery.channel_id, + true, + vec![delivery], + &mut by_event, + ); + assert_eq!(recovery.turn_id, "durable-panic-turn"); + assert!( + terminal.is_empty(), + "requeued panic must not terminalize the DB claim" + ); + assert_eq!(by_event[&event.id.to_hex()].claim_token, original_token); + assert!(queue.contains_event(channel_id, &event.id.to_hex())); + } } #[cfg(test)] diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 38749577398..bd79423e661 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -346,6 +346,10 @@ fn apply_completed_before_control_signal( pub enum ControlSignal { /// Stop the current turn and drop its triggering batch. Cancel, + /// Stop a turn whose workflow delivery entered terminal handling. Preserve + /// the triggering batch so the queue can remove only terminal workflow + /// events while retaining unrelated co-batched work. + TerminalWorkflowCancel, /// Stop the current turn and requeue its triggering batch for a merged /// re-prompt framed as a **supersede**: the new request replaces the old. Interrupt, @@ -759,7 +763,7 @@ impl AgentPool { &mut self, channel_id: Uuid, request: SteerRequest, - ) -> Result<(), SteerError> { + ) -> Result { let meta = self .task_map .values_mut() @@ -770,7 +774,8 @@ impl AgentPool { .as_ref() .ok_or_else(|| SteerError::Transport("steer_tx not installed".into()))?; tx.try_send(request) - .map_err(|e| SteerError::Transport(e.to_string())) + .map_err(|e| SteerError::Transport(e.to_string()))?; + Ok(meta.turn_id.clone()) } /// Durably associate a successful steer with the exact ACP session that @@ -3965,13 +3970,25 @@ fn requeue_cancelled_batch( signal: ControlSignal, batch: Option, ) -> Option { - let reason = match signal { - ControlSignal::Steer => CancelReason::Steer, - ControlSignal::Interrupt | ControlSignal::SwitchModel { .. } => CancelReason::Interrupt, + let (reason, preserve_in_drop) = match signal { + ControlSignal::Steer => (CancelReason::Steer, false), + // A terminal workflow cancellation is workflow-specific recovery, not + // a global change to Drop semantics. Preserve the checked-out batch so + // EventQueue can tombstone the terminal workflow event and retain only + // the remaining co-batched work. + ControlSignal::TerminalWorkflowCancel => (CancelReason::Interrupt, true), + ControlSignal::Interrupt | ControlSignal::SwitchModel { .. } => { + (CancelReason::Interrupt, false) + } // Cancel/Rotate discard the batch — no merged re-prompt. ControlSignal::Cancel | ControlSignal::Rotate => return None, }; - requeue_batch_if_queue(ctx, batch).map(|mut b| { + let batch = if preserve_in_drop { + batch + } else { + requeue_batch_if_queue(ctx, batch) + }; + batch.map(|mut b| { b.cancel_reason = Some(reason); b }) @@ -6855,10 +6872,40 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" } } + #[test] + fn terminal_workflow_cancel_is_the_only_drop_mode_cancel_that_preserves_batch() { + let mut ctx = make_prompt_context_no_owner(); + ctx.dedup_mode = DedupMode::Drop; + + let terminal = requeue_cancelled_batch( + &ctx, + ControlSignal::TerminalWorkflowCancel, + Some(one_event_batch(Uuid::new_v4())), + ) + .expect("terminal workflow cancellation must preserve its exact batch"); + assert_eq!(terminal.cancel_reason, Some(CancelReason::Interrupt)); + + for signal in [ControlSignal::Steer, ControlSignal::Interrupt] { + assert!( + requeue_cancelled_batch( + &ctx, + signal.clone(), + Some(one_event_batch(Uuid::new_v4())), + ) + .is_none(), + "ordinary {signal:?} must retain Drop semantics" + ); + } + } + #[test] fn test_requeue_cancelled_batch_maps_control_signal_to_cancel_reason() { let cases = [ (ControlSignal::Steer, Some(CancelReason::Steer)), + ( + ControlSignal::TerminalWorkflowCancel, + Some(CancelReason::Interrupt), + ), (ControlSignal::Interrupt, Some(CancelReason::Interrupt)), ( ControlSignal::SwitchModel { diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index 60866518bad..29c274cfe25 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -153,6 +153,10 @@ pub struct EventQueue { /// Set by `requeue_as_cancelled`, consumed by `flush_next` to set /// `FlushBatch::cancel_reason`. Keyed by channel, cleared on flush. cancel_reasons: HashMap, + /// Workflow events whose durable claims entered terminal handling while a + /// turn was in flight. They stay tombstoned until that turn returns so its + /// cancelled batch cannot resurrect them as ordinary prompts. + terminal_workflow_events: HashMap>, /// Events withheld from `queues` while a goose-native steer is in flight /// for that event. Invisible to `flush_next` / `has_flushable_work` / /// `drain` (the events have been moved out of `queues`), so the queue's @@ -187,6 +191,7 @@ impl EventQueue { dedup_mode, cancelled_batches: HashMap::new(), cancel_reasons: HashMap::new(), + terminal_workflow_events: HashMap::new(), withheld_native_steer: HashMap::new(), in_flight_deadline: Duration::from_secs(DEFAULT_IN_FLIGHT_DEADLINE_SECS), } @@ -278,6 +283,7 @@ impl EventQueue { ); self.in_flight_channels.remove(&id); self.in_flight_deadlines.remove(&id); + self.terminal_workflow_events.remove(&id); // Recover any withheld goose-native steer events for the expired // channel back to the queue front so normal dispatch delivers // them. Unlike the in-flight batch above (already delivered to a @@ -393,6 +399,10 @@ impl EventQueue { self.in_flight_channels.remove(&channel_id); self.in_flight_deadlines.remove(&channel_id); self.in_flight_batch_sizes.remove(&channel_id); + // `requeue_as_cancelled` runs before `mark_complete`; the returning + // batch has now crossed the tombstone and can no longer resurrect a + // terminal workflow event. + self.terminal_workflow_events.remove(&channel_id); let now = Instant::now(); match self.retry_after.get(&channel_id) { // Active throttle → channel was requeued; keep retry_counts intact. @@ -426,7 +436,11 @@ impl EventQueue { /// /// Note: does NOT remove from `in_flight_channels` — caller must call /// `mark_complete` separately. - pub fn requeue(&mut self, batch: FlushBatch) -> Option { + pub fn requeue(&mut self, mut batch: FlushBatch) -> Option { + self.filter_terminal_workflow_events(&mut batch); + if batch.events.is_empty() && batch.cancelled_events.is_empty() { + return None; + } let channel_id = batch.channel_id; let attempt = { let count = self.retry_counts.entry(channel_id).or_insert(0); @@ -474,7 +488,7 @@ impl EventQueue { let queue = self.queues.entry(channel_id).or_default(); // Push to front in reverse order so original order is preserved. - for be in batch.events.into_iter().rev() { + for be in batch.cancelled_events.into_iter().chain(batch.events).rev() { queue.push_front(QueuedEvent { channel_id, event: be.event, @@ -505,11 +519,12 @@ impl EventQueue { /// /// Does NOT set `retry_after`. Does NOT remove from `in_flight_channels` — /// caller must call `mark_complete` separately. - pub fn requeue_preserve_timestamps(&mut self, batch: FlushBatch) { + pub fn requeue_preserve_timestamps(&mut self, mut batch: FlushBatch) { + self.filter_terminal_workflow_events(&mut batch); let channel_id = batch.channel_id; let queue = self.queues.entry(channel_id).or_default(); // Push to front in reverse order so original order is preserved. - for be in batch.events.into_iter().rev() { + for be in batch.cancelled_events.into_iter().chain(batch.events).rev() { queue.push_front(QueuedEvent { channel_id, event: be.event, @@ -539,12 +554,18 @@ impl EventQueue { /// Unlike `requeue_preserve_timestamps`, events are NOT pushed back into /// the generic queue — they are stored separately and merged by /// `flush_next()`. No retry throttle, no backoff. - pub fn requeue_as_cancelled(&mut self, batch: FlushBatch, reason: CancelReason) { + pub fn requeue_as_cancelled(&mut self, mut batch: FlushBatch, reason: CancelReason) { + self.filter_terminal_workflow_events(&mut batch); let entry = self.cancelled_batches.entry(batch.channel_id).or_default(); // Preserve any already-cancelled events from a prior cancel (double-cancel). entry.extend(batch.cancelled_events); entry.extend(batch.events); - self.cancel_reasons.insert(batch.channel_id, reason); + if entry.is_empty() { + self.cancelled_batches.remove(&batch.channel_id); + self.cancel_reasons.remove(&batch.channel_id); + } else { + self.cancel_reasons.insert(batch.channel_id, reason); + } } /// Returns `true` if any channel has pending events that are not in-flight @@ -574,6 +595,7 @@ impl EventQueue { ); self.in_flight_channels.remove(&id); self.in_flight_deadlines.remove(&id); + self.terminal_workflow_events.remove(&id); // Symmetric with the flush_next expiry block: recover withheld // goose-native steer events for the expired channel so they are // not permanently orphaned in the side table. @@ -656,16 +678,19 @@ impl EventQueue { /// Returns the event IDs of dropped events so the caller can clean up /// any reactions (👀) that were added at queue-push time. pub fn drain_channel(&mut self, channel_id: Uuid) -> Vec { - let ids = self - .queues - .remove(&channel_id) - .map(|q| q.into_iter().map(|e| e.event.id.to_hex()).collect()) - .unwrap_or_default(); + let mut ids = Vec::new(); + if let Some(events) = self.queues.remove(&channel_id) { + ids.extend(events.into_iter().map(|event| event.event.id.to_hex())); + } + if let Some(events) = self.cancelled_batches.remove(&channel_id) { + ids.extend(events.into_iter().map(|event| event.event.id.to_hex())); + } + if let Some(events) = self.withheld_native_steer.remove(&channel_id) { + ids.extend(events.into_iter().map(|event| event.event.id.to_hex())); + } self.retry_after.remove(&channel_id); self.retry_counts.remove(&channel_id); - self.cancelled_batches.remove(&channel_id); self.cancel_reasons.remove(&channel_id); - self.withheld_native_steer.remove(&channel_id); // Preserve in_flight_channels AND in_flight_deadlines: the in-flight // task will eventually complete (calling mark_complete) or the deadline // will expire (auto-cleaning the channel). Removing deadlines without @@ -767,6 +792,66 @@ impl EventQueue { } } + /// Whether an event remains under local queue ownership, including a + /// cancel-merge batch or a pending native steer. + pub fn contains_event(&self, channel_id: Uuid, event_id: &str) -> bool { + self.queues.get(&channel_id).is_some_and(|events| { + events + .iter() + .any(|event| event.event.id.to_hex() == event_id) + }) || self + .cancelled_batches + .get(&channel_id) + .is_some_and(|events| { + events + .iter() + .any(|event| event.event.id.to_hex() == event_id) + }) + || self + .withheld_native_steer + .get(&channel_id) + .is_some_and(|events| { + events + .iter() + .any(|event| event.event.id.to_hex() == event_id) + }) + } + + /// Remove terminal workflow events from a returning or recovered batch. + /// Every batch-fate path must cross this authority before requeueing. + fn filter_terminal_workflow_events(&self, batch: &mut FlushBatch) { + let Some(terminal) = self.terminal_workflow_events.get(&batch.channel_id) else { + return; + }; + batch + .cancelled_events + .retain(|event| !terminal.contains(&event.event.id.to_hex())); + batch + .events + .retain(|event| !terminal.contains(&event.event.id.to_hex())); + } + + /// Remove a terminal workflow event from every queued store and keep it + /// tombstoned until the current turn returns. Ordinary cancellation keeps + /// its existing re-prompt behavior; only callers that have begun durable + /// terminal handling use this fence. + pub fn terminalize_workflow_event(&mut self, channel_id: Uuid, event_id: &str) { + self.remove_event(channel_id, event_id); + if let Some(events) = self.cancelled_batches.get_mut(&channel_id) { + events.retain(|event| event.event.id.to_hex() != event_id); + if events.is_empty() { + self.cancelled_batches.remove(&channel_id); + self.cancel_reasons.remove(&channel_id); + } + } + if self.in_flight_channels.contains(&channel_id) { + self.terminal_workflow_events + .entry(channel_id) + .or_default() + .insert(event_id.to_owned()); + } + } + /// Drop a specific event by id from both the side table and the main /// queue. /// @@ -3964,6 +4049,29 @@ mod tests { assert_eq!(pending_count(&q), 0); } + #[test] + fn test_drain_channel_reports_cancelled_and_withheld_events() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + let cancelled = make_queued(ch, "cancelled"); + let withheld = make_queued(ch, "withheld"); + let cancelled_id = cancelled.event.id.to_hex(); + let withheld_id = withheld.event.id.to_hex(); + + q.push(cancelled); + let batch = q.flush_next().expect("cancelled batch"); + q.requeue_as_cancelled(batch, CancelReason::Steer); + q.mark_complete(ch); + q.push(withheld); + assert!(q.mark_native_steer_pending(ch, &withheld_id)); + + let drained = q.drain_channel(ch); + assert!(drained.contains(&cancelled_id)); + assert!(drained.contains(&withheld_id)); + assert!(!q.contains_event(ch, &cancelled_id)); + assert!(!q.contains_event(ch, &withheld_id)); + } + #[test] fn test_drain_channel_empty_returns_empty() { let mut q = EventQueue::new(DedupMode::Queue); @@ -4095,6 +4203,121 @@ mod tests { ); } + #[test] + fn terminal_workflow_in_checked_out_batch_never_reflushes() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + let workflow = make_queued(ch, "terminal workflow"); + let workflow_id = workflow.event.id.to_hex(); + q.push(workflow); + q.push(make_queued(ch, "unrelated in flight")); + let batch = q.flush_next().expect("checked-out batch"); + q.push(make_queued(ch, "unrelated queued")); + q.terminalize_workflow_event(ch, &workflow_id); + q.requeue_as_cancelled(batch, CancelReason::Interrupt); + q.mark_complete(ch); + let next = q.flush_next().expect("unrelated events survive"); + let contents = next + .events + .iter() + .chain(&next.cancelled_events) + .map(|event| event.event.content.as_str()) + .collect::>(); + assert_eq!(contents.len(), 2); + assert!(contents.contains(&"unrelated in flight")); + assert!(contents.contains(&"unrelated queued")); + assert!(!contents.contains(&"terminal workflow")); + q.mark_complete(ch); + assert!(q.flush_next().is_none(), "terminal workflow cannot reflush"); + } + + #[test] + fn terminal_workflow_already_cancel_merged_never_reflushes() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + let workflow = make_queued(ch, "terminal workflow"); + let workflow_id = workflow.event.id.to_hex(); + q.push(workflow); + let first = q.flush_next().expect("first batch"); + q.push(make_queued(ch, "unrelated merged")); + q.requeue_as_cancelled(first, CancelReason::Interrupt); + q.mark_complete(ch); + let merged = q.flush_next().expect("cancel-merged batch"); + q.push(make_queued(ch, "unrelated queued")); + q.terminalize_workflow_event(ch, &workflow_id); + q.requeue_as_cancelled(merged, CancelReason::Interrupt); + q.mark_complete(ch); + let next = q.flush_next().expect("unrelated events survive"); + let contents = next + .events + .iter() + .chain(&next.cancelled_events) + .map(|event| event.event.content.as_str()) + .collect::>(); + assert_eq!(contents.len(), 2); + assert!(contents.contains(&"unrelated merged")); + assert!(contents.contains(&"unrelated queued")); + assert!(!contents.contains(&"terminal workflow")); + q.mark_complete(ch); + assert!(q.flush_next().is_none(), "terminal workflow cannot reflush"); + } + + #[test] + fn terminal_workflow_retry_filters_both_event_buckets() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + let workflow = make_queued(ch, "terminal workflow"); + let workflow_id = workflow.event.id.to_hex(); + q.push(workflow); + q.push(make_queued(ch, "unrelated event")); + let mut batch = q.flush_next().unwrap(); + batch.cancelled_events.push(BatchEvent { + event: make_event("unrelated cancelled"), + prompt_tag: "test".into(), + received_at: Instant::now(), + }); + batch.cancelled_events.push(batch.events[0].clone()); + q.terminalize_workflow_event(ch, &workflow_id); + assert!(q.requeue(batch).is_none()); + q.mark_complete(ch); + assert!(!q.contains_event(ch, &workflow_id)); + assert_eq!(q.queued_event_count(&ch), 2); + } + + #[test] + fn terminal_workflow_preserve_retry_filters_both_event_buckets() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + let workflow = make_queued(ch, "terminal workflow"); + let workflow_id = workflow.event.id.to_hex(); + q.push(workflow); + q.push(make_queued(ch, "unrelated event")); + let mut batch = q.flush_next().unwrap(); + batch.cancelled_events.push(BatchEvent { + event: make_event("unrelated cancelled"), + prompt_tag: "test".into(), + received_at: Instant::now(), + }); + batch.cancelled_events.push(batch.events[0].clone()); + q.terminalize_workflow_event(ch, &workflow_id); + q.requeue_preserve_timestamps(batch); + q.mark_complete(ch); + assert!(!q.contains_event(ch, &workflow_id)); + assert_eq!(q.queued_event_count(&ch), 2); + } + + #[test] + fn ordinary_cancellation_still_reflushes() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + q.push(make_queued(ch, "ordinary")); + let batch = q.flush_next().expect("ordinary batch"); + q.requeue_as_cancelled(batch, CancelReason::Interrupt); + q.mark_complete(ch); + let retry = q.flush_next().expect("ordinary cancellation reflushes"); + assert_eq!(retry.events[0].event.content, "ordinary"); + } + #[test] fn test_requeue_as_cancelled_propagates_reason() { let mut q = EventQueue::new(DedupMode::Queue); diff --git a/crates/buzz-acp/src/relay.rs b/crates/buzz-acp/src/relay.rs index 17a818867dd..245fbf4d51d 100644 --- a/crates/buzz-acp/src/relay.rs +++ b/crates/buzz-acp/src/relay.rs @@ -436,6 +436,17 @@ impl RestClient { .map_err(|e| RelayError::Http(e.to_string())) } + /// POST an authenticated JSON document to an arbitrary relay API path. + pub async fn post_json(&self, path: &str, value: &Value) -> Result { + let body = serde_json::to_vec(value) + .map_err(|error| RelayError::Http(format!("JSON serialize error: {error}")))?; + let response = self.bridge_post(path, &body).await?; + response + .json() + .await + .map_err(|error| RelayError::Http(error.to_string())) + } + /// Submit a signed event via the HTTP bridge: `POST /events` with NIP-98 auth. /// /// The event must already be signed. Returns the relay response JSON. @@ -577,6 +588,9 @@ pub struct HarnessRelay { keys: Keys, /// Optional NIP-OA auth tag for relay membership delegation. auth_tag: Option, + /// Relay signing identity advertised by NIP-11 `self`. Relay-authored + /// workflow messages are trusted only when their signature matches this key. + relay_self: Option, /// Handle to the background task (for clean shutdown). /// Wrapped in `Option` so `shutdown()` can take ownership without conflicting /// with `Drop` (which only has `&mut self`). @@ -630,6 +644,11 @@ impl HarnessRelay { agent_pubkey_hex: &str, auth_tag: Option, ) -> Result { + let http = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .connect_timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(|e| RelayError::Http(format!("failed to build HTTP client: {e}")))?; // Perform the initial connection and auth handshake, retrying // transient failures (dropped handshake, timeout) with bounded // jittered backoff. A terminal error (bad URL, bad auth tag, @@ -638,6 +657,18 @@ impl HarnessRelay { let (ws, handshake_buffer) = retry_initial_connect(|| do_connect(relay_url, keys, auth_tag.as_ref())).await?; + // NIP-11 is the authority binding between this configured endpoint and + // its relay signing key. Fetch it after the successful connection so a + // transient startup outage cannot permanently disable workflow handling. + // Missing, malformed, or unreachable info still fails closed for workflow + // delegation without preventing ordinary ACP use. + let relay_self = fetch_relay_self(&http, relay_url).await; + if relay_self.is_none() { + warn!( + "could not fetch relay NIP-11 identity after connecting; workflow delegation is disabled until restart" + ); + } + let (event_tx, event_rx) = mpsc::channel::>(event_channel_capacity()); let (observer_control_tx, observer_control_rx) = mpsc::channel::(event_channel_capacity()); @@ -667,14 +698,11 @@ impl HarnessRelay { event_rx, observer_control_rx: Some(observer_control_rx), cmd_tx, - http: reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .connect_timeout(std::time::Duration::from_secs(5)) - .build() - .map_err(|e| RelayError::Http(format!("failed to build HTTP client: {e}")))?, + http, relay_url: relay_url.to_string(), keys: keys.clone(), auth_tag, + relay_self, bg_handle: Some(bg_handle), }) } @@ -743,6 +771,11 @@ impl HarnessRelay { Ok(map) } + /// Return the configured endpoint's NIP-11 relay signing identity. + pub fn relay_self(&self) -> Option<&str> { + self.relay_self.as_deref() + } + /// Build a [`RestClient`] that shares this relay's HTTP credentials. /// /// The returned client is cheap to clone (wraps `reqwest::Client` which is @@ -3498,6 +3531,64 @@ async fn send_auth_response( /// `ws://host:port` → `http://host:port` /// `wss://host:port` → `https://host:port` /// Trailing slashes are stripped. +#[derive(serde::Deserialize)] +struct RelayInformationDocument { + #[serde(default, rename = "self")] + relay_self: Option, +} + +async fn fetch_relay_self(http: &reqwest::Client, relay_url: &str) -> Option { + // Keep NIP-11 retrieval bounded. The initial connection already proved the + // endpoint reachable; these retries cover a short-lived HTTP-side outage. + for (attempt, delay) in std::iter::once(None) + .chain( + STARTUP_CONNECT_BACKOFFS + .iter() + .take(2) + .map(|delay| Some(*delay)), + ) + .enumerate() + { + if let Some(delay) = delay { + tokio::time::sleep(jittered_duration(delay)).await; + } + + let Some(response) = http + .get(relay_ws_to_http(relay_url)) + .header("Accept", "application/nostr+json") + .send() + .await + .ok() + else { + warn!("NIP-11 relay identity fetch attempt {attempt} failed"); + continue; + }; + if !response.status().is_success() { + warn!( + "NIP-11 relay identity fetch attempt {attempt} returned HTTP {}", + response.status() + ); + continue; + } + let Some(value) = response + .json::() + .await + .ok() + .and_then(|document| document.relay_self) + .map(|value| value.to_ascii_lowercase()) + else { + warn!("NIP-11 relay identity fetch attempt {attempt} returned invalid metadata"); + continue; + }; + if let Ok(key) = nostr::PublicKey::from_hex(&value) { + return Some(key.to_hex()); + } + warn!("NIP-11 relay identity fetch attempt {attempt} returned an invalid public key"); + } + + None +} + pub(crate) fn relay_ws_to_http(url: &str) -> String { url.replace("wss://", "https://") .replace("ws://", "http://") diff --git a/crates/buzz-core/src/kind.rs b/crates/buzz-core/src/kind.rs index 3c6f1d5913d..a5d7609f42f 100644 --- a/crates/buzz-core/src/kind.rs +++ b/crates/buzz-core/src/kind.rs @@ -158,6 +158,7 @@ pub const RESULT_GATED_KINDS: &[u32] = &[KIND_DM_VISIBILITY, KIND_AGENT_TURN_MET /// storage-layer search defense does not apply to them. pub const P_GATED_KINDS: &[u32] = &[ KIND_AGENT_OBSERVER_FRAME, + KIND_WORKFLOW_AGENT_WAKE, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_GIFT_WRAP, @@ -467,6 +468,10 @@ pub const KIND_PAIRING: u32 = 24134; pub const KIND_TYPING_INDICATOR: u32 = 20002; /// Ephemeral: owner-scoped encrypted agent observer telemetry and control frame. pub const KIND_AGENT_OBSERVER_FRAME: u32 = 24200; +/// Ephemeral relay-signed wake for a workflow message addressed to an agent. +/// The authored message remains an ordinary persisted kind:9; this event only +/// points at verified workflow execution state needed to admit that message. +pub const KIND_WORKFLOW_AGENT_WAKE: u32 = 24620; /// Ephemeral: huddle emoji reaction burst. Channel-scoped to the ephemeral /// huddle channel with an `h` tag; never stored in the timeline. pub const KIND_HUDDLE_REACTION: u32 = 24810; @@ -834,6 +839,7 @@ pub const fn is_relay_only_kind(kind: u32) -> bool { | KIND_CHANNEL_SUMMARY | KIND_PRESENCE_SNAPSHOT | KIND_DM_VISIBILITY + | KIND_WORKFLOW_AGENT_WAKE | KIND_THREAD_SUMMARY | KIND_WINDOW_BOUNDS ) diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index 36dc772da3b..58cf52286cc 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -42,6 +42,8 @@ pub mod relay; pub mod tenant; /// Schnorr signature and event ID verification. pub mod verification; +/// Durable workflow-delivery lease bounds shared by clients and the relay. +pub mod workflow_delivery; pub use error::VerificationError; pub use event::StoredEvent; diff --git a/crates/buzz-core/src/workflow_delivery.rs b/crates/buzz-core/src/workflow_delivery.rs new file mode 100644 index 00000000000..f58fc94763f --- /dev/null +++ b/crates/buzz-core/src/workflow_delivery.rs @@ -0,0 +1,16 @@ +//! Shared bounds for exclusive durable workflow-delivery leases. + +/// Default lease used by clients that do not request an explicit duration. +pub const DEFAULT_LEASE_SECONDS: i64 = 120; + +/// Maximum lease accepted by the relay. +/// +/// This covers buzz-acp's supported seven-day turn ceiling plus its 400-second +/// local retry and handoff margin. +pub const MAX_LEASE_SECONDS: i64 = 604_800 + 400; + +/// Lifetime of a durable delivery row. +/// +/// The extra day permits delayed polling before a maximum-duration lease is +/// claimed while still leaving the complete supported turn window available. +pub const ROW_LIFETIME_SECONDS: i64 = MAX_LEASE_SECONDS + 24 * 60 * 60; diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index fbe69f22a68..60f518bffc1 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -79,6 +79,7 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "subscriptions", "thread_metadata", "users", + "workflow_agent_deliveries", "workflow_approvals", "workflow_runs", "workflows", @@ -86,6 +87,7 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ /// Foreign-key-safe child-before-parent order for the PostgreSQL purge. pub const PURGE_SCOPED_TABLES: &[&str] = &[ + "workflow_agent_deliveries", "workflow_approvals", "scheduled_workflow_fires", "workflow_runs", diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 3ff230f9503..1b101d0536f 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -3864,6 +3864,8 @@ impl Db { name: &str, definition_json: &str, definition_hash: &[u8], + definition_event_id: &[u8], + enabled: bool, ) -> Result<()> { workflow::upsert_workflow( &self.pool, @@ -3874,6 +3876,8 @@ impl Db { name, definition_json, definition_hash, + definition_event_id, + enabled, ) .await } @@ -4130,6 +4134,188 @@ impl Db { .await } + /// Serialize one durable delivery identity and return its canonical message. + pub async fn lock_workflow_agent_delivery_identity( + &self, + community_id: CommunityId, + run_id: Uuid, + step_id: &str, + target_pubkey: &[u8], + ) -> Result<( + sqlx::Transaction<'static, sqlx::Postgres>, + Option<(Vec, chrono::DateTime)>, + )> { + workflow::lock_workflow_agent_delivery_identity( + &self.pool, + community_id, + run_id, + step_id, + target_pubkey, + ) + .await + } + + /// Atomically persist a workflow message and reconcile all signed targets. + #[allow(clippy::too_many_arguments)] + pub async fn commit_workflow_agent_deliveries( + &self, + transaction: sqlx::Transaction<'static, sqlx::Postgres>, + community_id: CommunityId, + event: Option<&nostr::Event>, + message_event_id: &[u8], + message_event_created_at: chrono::DateTime, + thread_meta: Option>, + workflow_id: Uuid, + run_id: Uuid, + step_id: &str, + definition_event_id: &[u8], + channel_id: Uuid, + targets: &[workflow::WorkflowAgentDeliveryTarget], + execution_trace: &serde_json::Value, + trigger_context: Option<&serde_json::Value>, + expires_at: chrono::DateTime, + ) -> Result<(Option, Vec)> { + let result = workflow::commit_workflow_agent_deliveries( + transaction, + community_id, + event, + message_event_id, + message_event_created_at, + thread_meta, + workflow_id, + run_id, + step_id, + definition_event_id, + channel_id, + targets, + execution_trace, + trigger_context, + expires_at, + ) + .await?; + if result.0.is_some() { + if let Some(event) = event { + if let Err(error) = + insert_mentions(&self.pool, community_id, event, Some(channel_id)).await + { + tracing::warn!(event_id = %event.id, %error, "failed to insert workflow message mentions"); + } + } + } + Ok(result) + } + + /// Create one durable managed-agent workflow delivery. + #[allow(clippy::too_many_arguments)] + pub async fn create_workflow_agent_delivery( + &self, + community_id: CommunityId, + id: Uuid, + workflow_id: Uuid, + run_id: Uuid, + step_id: &str, + definition_event_id: &[u8], + message_event_id: &[u8], + message_event_created_at: chrono::DateTime, + channel_id: Uuid, + target_pubkey: &[u8], + execution_trace: &serde_json::Value, + trigger_context: Option<&serde_json::Value>, + expires_at: chrono::DateTime, + ) -> Result { + workflow::create_workflow_agent_delivery( + &self.pool, + community_id, + id, + workflow_id, + run_id, + step_id, + definition_event_id, + message_event_id, + message_event_created_at, + channel_id, + target_pubkey, + execution_trace, + trigger_context, + expires_at, + ) + .await + } + + /// Claim the oldest due delivery for this managed agent. + pub async fn claim_workflow_agent_delivery( + &self, + community_id: CommunityId, + target_pubkey: &[u8], + delivery_id: Option, + expected: Option<&workflow::WorkflowAgentDeliveryBinding>, + lease_seconds: i64, + ) -> Result> { + workflow::claim_workflow_agent_delivery( + &self.pool, + community_id, + target_pubkey, + delivery_id, + expected, + lease_seconds, + ) + .await + } + + /// Extend a live delivery claim using its owner/token fencing stamp. + pub async fn renew_workflow_agent_delivery( + &self, + community_id: CommunityId, + id: uuid::Uuid, + target_pubkey: &[u8], + claim_token: uuid::Uuid, + lease_seconds: i64, + ) -> Result>> { + workflow::renew_workflow_agent_delivery( + &self.pool, + community_id, + id, + target_pubkey, + claim_token, + lease_seconds, + ) + .await + } + + /// Fence and complete one claimed delivery. + #[allow(clippy::too_many_arguments)] + pub async fn finish_workflow_agent_delivery( + &self, + community_id: CommunityId, + id: Uuid, + target_pubkey: &[u8], + claim_token: Uuid, + delivered: bool, + retryable: bool, + failure_code: Option<&str>, + failure_message: Option<&str>, + ) -> Result { + workflow::finish_workflow_agent_delivery( + &self.pool, + community_id, + id, + target_pubkey, + claim_token, + delivered, + retryable, + failure_code, + failure_message, + ) + .await + } + + /// Mark expired or retry-exhausted managed-agent deliveries failed. + pub async fn reap_workflow_agent_deliveries( + &self, + ) -> Result> { + workflow::reap_workflow_agent_deliveries(&self.pool).await + } + /// Update a workflow run's status. #[datastore_span(name = "update_workflow_run", system = "postgresql")] pub async fn update_workflow_run( diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 94c7aea2faf..a7638a31c5e 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -640,7 +640,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 32); + assert_eq!(migrations.len(), 34); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1067,6 +1067,34 @@ mod tests { assert!(roster_fence.contains("snapshot_members IS DISTINCT FROM canonical_members")); assert!(roster_fence.contains("ERRCODE = '23514'")); + // Durable workflow-agent delivery schema must remain byte-for-byte equivalent at the + // statement level between migration and desired-state bootstrap. A fresh pgschema + // database does not run migration 0034, so drift here silently breaks the live path. + assert_eq!(migrations[33].version, 34); + let delivery_migration = migrations[33].sql.as_str(); + for prefix in [ + "CREATE TYPE workflow_agent_delivery_status", + "CREATE TABLE workflow_agent_deliveries", + "CREATE INDEX idx_workflow_agent_deliveries_pending", + "CREATE INDEX idx_workflow_agent_deliveries_run", + ] { + let from_migration = split_sql_statements(delivery_migration) + .into_iter() + .find(|statement| statement.trim_start().starts_with(prefix)) + .unwrap_or_else(|| panic!("migration 0034 is missing {prefix}")); + let from_schema = split_sql_statements(desired_schema) + .into_iter() + .find(|statement| statement.trim_start().starts_with(prefix)) + .unwrap_or_else(|| panic!("schema.sql is missing {prefix}")); + assert_eq!( + normalize_sql(&from_schema), + normalize_sql(&from_migration), + "desired-state {prefix} drifted from migration 0034" + ); + } + assert!(desired_schema + .contains("SELECT attach_community_write_fence('workflow_agent_deliveries')")); + // Fresh desired-state bootstrap must install the identical executable // fence as migration 0032. CI and isolated relay startup use schema.sql // without running migrations, so drift reopens rolling-deploy races. @@ -1542,6 +1570,8 @@ mod tests { let mut expected_fences = migration.fence_attachments.clone(); expected_fences.remove("product_feedback"); expected_fences.remove("rate_limit_violations"); + // Added by migration 0034 and therefore absent from migration 0029. + expected_fences.insert("workflow_agent_deliveries".to_string()); assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" @@ -1968,6 +1998,37 @@ mod tests { assert_eq!(after, vec![(1, Some(true)), (30_350, None)]); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn workflow_agent_delivery_migration_binds_the_partitioned_event_key() { + let pool = connect_test_pool().await; + reset_public_schema(&pool).await; + + MIGRATOR + .run_to(33, &pool) + .await + .expect("apply migrations through workflow definition event identity"); + MIGRATOR + .run_to(34, &pool) + .await + .expect("apply workflow agent delivery migration"); + + assert_eq!(applied_versions(&pool).await.last(), Some(&34)); + let foreign_key: String = sqlx::query_scalar( + "SELECT pg_get_constraintdef(oid) FROM pg_constraint \ + WHERE conrelid = 'workflow_agent_deliveries'::regclass \ + AND contype = 'f' AND confrelid = 'events'::regclass", + ) + .fetch_one(&pool) + .await + .expect("read delivery-to-event foreign key"); + assert_eq!( + foreign_key, + "FOREIGN KEY (community_id, message_event_created_at, message_event_id) \ + REFERENCES events(community_id, created_at, id) ON DELETE CASCADE" + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn run_migrations_applies_consolidated_initial_schema_on_fresh_database() { diff --git a/crates/buzz-db/src/workflow.rs b/crates/buzz-db/src/workflow.rs index e970e978aaf..48c83ba897b 100644 --- a/crates/buzz-db/src/workflow.rs +++ b/crates/buzz-db/src/workflow.rs @@ -12,10 +12,13 @@ use std::str::FromStr; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; -use sqlx::{PgPool, Row}; +use sqlx::{PgPool, Postgres, Row, Transaction}; use uuid::Uuid; -use buzz_core::CommunityId; +use buzz_core::{CommunityId, StoredEvent}; +use nostr::Event; + +use crate::event::ThreadMetadataParams; use crate::error::{DbError, Result}; @@ -177,6 +180,8 @@ pub struct WorkflowRecord { pub definition: serde_json::Value, /// SHA-256 hash of the canonical definition JSON. pub definition_hash: Vec, + /// Exact owner-signed kind:30620 event that materialized this revision. + pub definition_event_id: Option>, /// Current lifecycle status of the workflow definition. pub status: WorkflowStatus, /// Whether the workflow will fire on matching events. @@ -225,6 +230,43 @@ pub struct WorkflowRunRecord { pub created_at: DateTime, } +/// A workflow message awaiting or completing managed-agent delivery. +#[derive(Debug, Clone)] +pub struct WorkflowAgentDeliveryRecord { + /// Stable delivery identifier referenced by private wake hints. + pub id: Uuid, + /// Community that owns this delivery. + pub community_id: CommunityId, + /// Workflow whose action created the delivery. + pub workflow_id: Uuid, + /// Execution run containing the action. + pub run_id: Uuid, + /// Stable step identifier within the workflow definition. + pub step_id: String, + /// Exact signed workflow-definition event required for admission. + pub definition_event_id: Vec, + /// Exact persisted visible message event required for admission. + pub message_event_id: Vec, + /// Channel shared by the definition and visible message. + pub channel_id: Uuid, + /// Managed agent allowed to claim and complete this delivery. + pub target_pubkey: Vec, + /// Durable delivery state. + pub status: String, + /// Number of leases issued, including the current lease when claimed. + pub attempt: i32, + /// Fencing token for the current lease, if claimed. + pub claim_token: Option, + /// Expiry of the current claim lease, if claimed. + pub claim_expires_at: Option>, + /// Absolute delivery expiry after which no new lease may be issued. + pub expires_at: DateTime, + /// Immutable execution state used to verify rendering and prior-step output. + pub execution_trace: serde_json::Value, + /// Immutable trigger input, including private webhook fields when present. + pub trigger_context: Option, +} + /// A winning scheduled workflow fire claim. /// /// The primary identity is `(workflow_id, scheduled_for)`. `community_id` is @@ -322,16 +364,20 @@ pub async fn upsert_workflow( name: &str, definition_json: &str, definition_hash: &[u8], + definition_event_id: &[u8], + enabled: bool, ) -> Result<()> { let row = sqlx::query( r#" INSERT INTO workflows - (community_id, id, name, owner_pubkey, channel_id, definition, definition_hash, status, enabled) - VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, 'active', TRUE) + (community_id, id, name, owner_pubkey, channel_id, definition, definition_hash, definition_event_id, status, enabled) + VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, 'active', $9) ON CONFLICT (community_id, id) DO UPDATE SET name = EXCLUDED.name, definition = EXCLUDED.definition, definition_hash = EXCLUDED.definition_hash, + definition_event_id = EXCLUDED.definition_event_id, + enabled = EXCLUDED.enabled, updated_at = NOW() WHERE workflows.owner_pubkey = EXCLUDED.owner_pubkey AND workflows.channel_id IS NOT DISTINCT FROM EXCLUDED.channel_id @@ -345,6 +391,8 @@ pub async fn upsert_workflow( .bind(channel_id) .bind(definition_json) .bind(definition_hash) + .bind(definition_event_id) + .bind(enabled) .fetch_optional(pool) .await?; @@ -370,7 +418,7 @@ pub async fn get_workflow( ) -> Result { let row = sqlx::query( r#" - SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, + SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, definition_event_id, status::text AS status, enabled, created_at, updated_at FROM workflows WHERE community_id = $1 AND id = $2 @@ -401,7 +449,7 @@ pub async fn list_channel_workflows( let rows = sqlx::query( r#" - SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, + SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, definition_event_id, status::text AS status, enabled, created_at, updated_at FROM workflows WHERE community_id = $1 AND channel_id = $2 @@ -432,7 +480,7 @@ pub async fn list_enabled_channel_workflows( ) -> Result> { let rows = sqlx::query( r#" - SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, + SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, definition_event_id, status::text AS status, enabled, created_at, updated_at FROM workflows WHERE community_id = $1 @@ -460,7 +508,7 @@ pub async fn list_enabled_channel_workflows( pub async fn list_all_enabled_workflows(pool: &PgPool) -> Result> { let rows = sqlx::query( r#" - SELECT w.id, w.community_id, w.name, w.owner_pubkey, w.channel_id, w.definition, w.definition_hash, + SELECT w.id, w.community_id, w.name, w.owner_pubkey, w.channel_id, w.definition, w.definition_hash, w.definition_event_id, w.status::text AS status, w.enabled, w.created_at, w.updated_at FROM workflows w JOIN communities c ON c.id = w.community_id @@ -961,6 +1009,409 @@ pub async fn update_workflow_run( Ok(()) } +/// Serialize creation of one durable delivery identity and return its canonical message. +/// +/// The transaction-scoped advisory lock remains held by the returned transaction. +/// Callers keep it alive until the visible event and durable row are published, so +/// concurrent retries cannot both pass the preflight and sign duplicate events. +pub async fn lock_workflow_agent_delivery_identity( + pool: &PgPool, + community_id: CommunityId, + run_id: Uuid, + step_id: &str, + _target_pubkey: &[u8], +) -> Result<( + Transaction<'static, Postgres>, + Option<(Vec, DateTime)>, +)> { + let mut transaction = pool.begin().await?; + let identity = format!("{}:{run_id}:{step_id}", community_id.as_uuid(),); + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(identity) + .execute(&mut *transaction) + .await?; + let message_event_id = sqlx::query_as::<_, (Vec, DateTime)>( + r#" + SELECT message_event_id, message_event_created_at + FROM workflow_agent_deliveries + WHERE community_id = $1 AND run_id = $2 AND step_id = $3 + ORDER BY created_at, id LIMIT 1 + "#, + ) + .bind(community_id.as_uuid()) + .bind(run_id) + .bind(step_id) + .fetch_optional(&mut *transaction) + .await?; + Ok((transaction, message_event_id)) +} + +/// One target in an atomic visible-message delivery commit. +pub struct WorkflowAgentDeliveryTarget { + /// Stable delivery identity used by wake hints and claim fencing. + pub id: Uuid, + /// Immutable managed-agent recipient. + pub pubkey: Vec, +} + +/// Atomically persist the canonical visible event and every signed routing target. +#[allow(clippy::too_many_arguments)] +pub async fn commit_workflow_agent_deliveries( + mut transaction: Transaction<'static, Postgres>, + community_id: CommunityId, + event: Option<&Event>, + message_event_id: &[u8], + message_event_created_at: DateTime, + thread_meta: Option>, + workflow_id: Uuid, + run_id: Uuid, + step_id: &str, + definition_event_id: &[u8], + channel_id: Uuid, + targets: &[WorkflowAgentDeliveryTarget], + execution_trace: &serde_json::Value, + trigger_context: Option<&serde_json::Value>, + expires_at: DateTime, +) -> Result<(Option, Vec)> { + let stored_event = if let Some(event) = event { + let (stored, _) = crate::event::insert_event_with_thread_metadata_tx( + &mut transaction, + community_id, + event, + Some(channel_id), + thread_meta, + ) + .await?; + Some(stored) + } else { + None + }; + let mut created = Vec::new(); + for target in targets { + let affected = sqlx::query( + r#"INSERT INTO workflow_agent_deliveries + (community_id, id, workflow_id, run_id, step_id, definition_event_id, + message_event_id, message_event_created_at, channel_id, target_pubkey, + execution_trace, trigger_context, expires_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) + ON CONFLICT (community_id, run_id, step_id, target_pubkey) DO NOTHING"#, + ) + .bind(community_id.as_uuid()) + .bind(target.id) + .bind(workflow_id) + .bind(run_id) + .bind(step_id) + .bind(definition_event_id) + .bind(message_event_id) + .bind(message_event_created_at) + .bind(channel_id) + .bind(&target.pubkey) + .bind(execution_trace) + .bind(trigger_context) + .bind(expires_at) + .execute(&mut *transaction) + .await? + .rows_affected(); + if affected == 1 { + created.push(target.id); + } + } + transaction.commit().await?; + Ok((stored_event, created)) +} + +/// Create one durable agent delivery after its immutable visible event exists. +#[allow(clippy::too_many_arguments)] +pub async fn create_workflow_agent_delivery( + pool: &PgPool, + community_id: CommunityId, + id: Uuid, + workflow_id: Uuid, + run_id: Uuid, + step_id: &str, + definition_event_id: &[u8], + message_event_id: &[u8], + message_event_created_at: DateTime, + channel_id: Uuid, + target_pubkey: &[u8], + execution_trace: &serde_json::Value, + trigger_context: Option<&serde_json::Value>, + expires_at: DateTime, +) -> Result { + let affected = sqlx::query( + r#" + INSERT INTO workflow_agent_deliveries + (community_id, id, workflow_id, run_id, step_id, definition_event_id, + message_event_id, message_event_created_at, channel_id, target_pubkey, execution_trace, trigger_context, expires_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) + ON CONFLICT (community_id, run_id, step_id, target_pubkey) DO NOTHING + "#, + ) + .bind(community_id.as_uuid()) + .bind(id) + .bind(workflow_id) + .bind(run_id) + .bind(step_id) + .bind(definition_event_id) + .bind(message_event_id) + .bind(message_event_created_at) + .bind(channel_id) + .bind(target_pubkey) + .bind(execution_trace) + .bind(trigger_context) + .bind(expires_at) + .execute(pool) + .await? + .rows_affected(); + Ok(affected == 1) +} + +/// Immutable wake bindings that must match before a specific live delivery is claimed. +#[derive(Debug, Clone)] +pub struct WorkflowAgentDeliveryBinding { + /// Run identifier carried by the authenticated wake. + pub run_id: Uuid, + /// Step identifier carried by the authenticated wake. + pub step_id: String, + /// Definition event identifier carried by the authenticated wake. + pub definition_event_id: Vec, + /// Visible message event identifier carried by the authenticated wake. + pub message_event_id: Vec, + /// Receiving channel carried by the authenticated wake. + pub channel_id: Uuid, +} + +/// Atomically acquire one due pending delivery for an agent. +/// +/// The requested lease must fit entirely within the row lifetime. Once claimed, +/// delivery retry belongs exclusively to the ACP runtime; an expired claim is +/// terminalized by the reaper rather than reacquired by another runtime. +pub async fn claim_workflow_agent_delivery( + pool: &PgPool, + community_id: CommunityId, + target_pubkey: &[u8], + delivery_id: Option, + expected: Option<&WorkflowAgentDeliveryBinding>, + lease_seconds: i64, +) -> Result> { + let row = sqlx::query( + r#" + WITH candidate AS ( + SELECT community_id, id + FROM workflow_agent_deliveries + WHERE community_id = $1 AND target_pubkey = $2 + AND ($3::uuid IS NULL OR id = $3::uuid) + AND ($5::uuid IS NULL OR run_id = $5::uuid) + AND ($6::text IS NULL OR step_id = $6::text) + AND ($7::bytea IS NULL OR definition_event_id = $7::bytea) + AND ($8::bytea IS NULL OR message_event_id = $8::bytea) + AND ($9::uuid IS NULL OR channel_id = $9::uuid) + AND expires_at >= NOW() + make_interval(secs => $4) + AND next_attempt_at <= NOW() + AND attempt < 3 + AND status = 'pending' + ORDER BY created_at, id + FOR UPDATE SKIP LOCKED + LIMIT 1 + ), claimed AS ( + UPDATE workflow_agent_deliveries d + SET status = 'claimed', + attempt = d.attempt + 1, + claim_token = gen_random_uuid(), claim_owner = $2, + claim_expires_at = NOW() + make_interval(secs => $4), updated_at = NOW() + FROM candidate c + WHERE d.community_id = c.community_id AND d.id = c.id + RETURNING d.* + ) + SELECT c.*, c.status::text AS status_text + FROM claimed c + "#, + ) + .bind(community_id.as_uuid()) + .bind(target_pubkey) + .bind(delivery_id) + .bind(lease_seconds as f64) + .bind(expected.map(|binding| binding.run_id)) + .bind(expected.map(|binding| binding.step_id.as_str())) + .bind(expected.map(|binding| binding.definition_event_id.as_slice())) + .bind(expected.map(|binding| binding.message_event_id.as_slice())) + .bind(expected.map(|binding| binding.channel_id)) + .fetch_optional(pool) + .await?; + row.map(row_to_agent_delivery_record).transpose() +} + +/// Extend a live delivery claim. The owner and token fence stale renewals. +pub async fn renew_workflow_agent_delivery( + pool: &PgPool, + community_id: CommunityId, + id: Uuid, + target_pubkey: &[u8], + claim_token: Uuid, + lease_seconds: i64, +) -> Result>> { + sqlx::query_scalar( + r#" + UPDATE workflow_agent_deliveries + SET claim_expires_at = LEAST( + expires_at, + GREATEST(claim_expires_at, NOW() + make_interval(secs => $1)) + ), + updated_at = NOW() + WHERE community_id = $2 AND id = $3 AND target_pubkey = $4 + AND claim_owner = $4 AND status = 'claimed' + AND claim_token = $5 AND claim_expires_at > NOW() + RETURNING claim_expires_at + "#, + ) + .bind(lease_seconds as f64) + .bind(community_id.as_uuid()) + .bind(id) + .bind(target_pubkey) + .bind(claim_token) + .fetch_optional(pool) + .await + .map_err(Into::into) +} + +/// Finish a claimed delivery. The token fences late/concurrent completions. +#[allow(clippy::too_many_arguments)] +pub async fn finish_workflow_agent_delivery( + pool: &PgPool, + community_id: CommunityId, + id: Uuid, + target_pubkey: &[u8], + claim_token: Uuid, + delivered: bool, + retryable: bool, + failure_code: Option<&str>, + failure_message: Option<&str>, +) -> Result { + let affected = sqlx::query( + r#" + UPDATE workflow_agent_deliveries + SET status = CASE + WHEN $1 THEN 'delivered'::workflow_agent_delivery_status + WHEN $2 AND attempt < 3 AND expires_at > NOW() + THEN 'pending'::workflow_agent_delivery_status + ELSE 'failed'::workflow_agent_delivery_status + END, + delivered_at = CASE WHEN $1 THEN NOW() ELSE delivered_at END, + failed_at = CASE WHEN NOT $1 AND NOT ($2 AND attempt < 3 AND expires_at > NOW()) + THEN NOW() ELSE failed_at END, + next_attempt_at = CASE WHEN NOT $1 AND $2 AND attempt < 3 AND expires_at > NOW() + THEN NOW() + make_interval(secs => LEAST(300, 5 * power(2, attempt - 1))::double precision) + ELSE next_attempt_at END, + claim_token = CASE + WHEN NOT $1 AND $2 AND attempt < 3 AND expires_at > NOW() THEN NULL + ELSE claim_token + END, + claim_owner = CASE + WHEN NOT $1 AND $2 AND attempt < 3 AND expires_at > NOW() THEN NULL + ELSE claim_owner + END, + claim_expires_at = CASE + WHEN NOT $1 AND $2 AND attempt < 3 AND expires_at > NOW() THEN NULL + ELSE claim_expires_at + END, + failure_code = $3, failure_message = $4, updated_at = NOW() + WHERE community_id = $5 AND id = $6 AND target_pubkey = $7 + AND claim_owner = $7 AND status = 'claimed' + AND claim_token = $8 AND claim_expires_at > NOW() + "#, + ) + .bind(delivered) + .bind(retryable) + .bind(failure_code) + .bind(failure_message) + .bind(community_id.as_uuid()) + .bind(id) + .bind(target_pubkey) + .bind(claim_token) + .execute(pool) + .await? + .rows_affected(); + if affected == 1 { + return Ok(true); + } + + // A successful terminal update may race a lost HTTP response. Keep the + // fencing token on terminal rows and accept only an exact replay of the + // same disposition; a stale owner can never change the recorded outcome. + let replayed = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 FROM workflow_agent_deliveries + WHERE community_id = $1 AND id = $2 AND target_pubkey = $3 + AND claim_owner = $3 AND claim_token = $4 + AND status = CASE WHEN $5 THEN 'delivered'::workflow_agent_delivery_status + ELSE 'failed'::workflow_agent_delivery_status END + AND ($5 OR (failure_code IS NOT DISTINCT FROM $6 + AND failure_message IS NOT DISTINCT FROM $7)) + ) + "#, + ) + .bind(community_id.as_uuid()) + .bind(id) + .bind(target_pubkey) + .bind(claim_token) + .bind(delivered) + .bind(failure_code) + .bind(failure_message) + .fetch_one(pool) + .await?; + Ok(replayed) +} + +/// Terminalize rows whose delivery lifetime or exclusive claim has expired. +pub async fn reap_workflow_agent_deliveries( + pool: &PgPool, +) -> Result> { + sqlx::query( + r#" + WITH terminal AS ( + UPDATE workflow_agent_deliveries + SET status = CASE WHEN expires_at <= NOW() THEN 'expired'::workflow_agent_delivery_status ELSE 'failed'::workflow_agent_delivery_status END, + failed_at = NOW(), failure_code = CASE WHEN expires_at <= NOW() THEN 'delivery_expired' ELSE 'delivery_claim_expired' END, + failure_message = CASE WHEN expires_at <= NOW() + THEN 'managed agent did not claim the workflow delivery' + ELSE 'managed agent lost exclusive workflow delivery ownership' END, + updated_at = NOW() + WHERE (status IN ('pending','claimed') AND expires_at <= NOW()) + OR (status = 'claimed' AND claim_expires_at <= NOW()) + RETURNING * + ) + SELECT t.*, t.status::text AS status_text FROM terminal t + "#, + ) + .fetch_all(pool) + .await? + .into_iter() + .map(row_to_agent_delivery_record) + .collect() +} + +fn row_to_agent_delivery_record(row: sqlx::postgres::PgRow) -> Result { + Ok(WorkflowAgentDeliveryRecord { + id: row.try_get("id")?, + community_id: CommunityId::from_uuid(row.try_get("community_id")?), + workflow_id: row.try_get("workflow_id")?, + run_id: row.try_get("run_id")?, + step_id: row.try_get("step_id")?, + definition_event_id: row.try_get("definition_event_id")?, + message_event_id: row.try_get("message_event_id")?, + channel_id: row.try_get("channel_id")?, + target_pubkey: row.try_get("target_pubkey")?, + status: row.try_get("status_text")?, + attempt: row.try_get("attempt")?, + claim_token: row.try_get("claim_token")?, + claim_expires_at: row.try_get("claim_expires_at")?, + expires_at: row.try_get("expires_at")?, + execution_trace: row.try_get("execution_trace")?, + trigger_context: row.try_get("trigger_context")?, + }) +} + // -- Approval CRUD ------------------------------------------------------------ /// Parameters for creating a new approval request. @@ -1183,6 +1634,7 @@ fn row_to_workflow_record(row: sqlx::postgres::PgRow) -> Result channel_id, definition: row.try_get("definition")?, definition_hash: row.try_get("definition_hash")?, + definition_event_id: row.try_get("definition_event_id")?, status, enabled, created_at: row.try_get("created_at")?, @@ -1247,7 +1699,7 @@ pub async fn find_by_owner_and_name( ) -> Result> { let row = sqlx::query( r#" - SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, + SELECT id, community_id, name, owner_pubkey, channel_id, definition, definition_hash, definition_event_id, status::text AS status, enabled, created_at, updated_at FROM workflows WHERE community_id = $1 AND owner_pubkey = $2 AND name = $3 @@ -1381,6 +1833,7 @@ mod tests { owner_pubkey: vec![0xab; 32], channel_id: Some(channel_id), definition: def.clone(), + definition_event_id: None, definition_hash: vec![0x01, 0x02, 0x03, 0x04], status: WorkflowStatus::Active, enabled: true, @@ -1411,6 +1864,7 @@ mod tests { owner_pubkey: vec![0x00; 32], channel_id: None, definition: serde_json::json!({}), + definition_event_id: None, definition_hash: vec![], status: WorkflowStatus::Active, enabled: true, @@ -1433,6 +1887,7 @@ mod tests { owner_pubkey: vec![0x01; 32], channel_id: None, definition: serde_json::json!({}), + definition_event_id: None, definition_hash: vec![0xAA], status: WorkflowStatus::Active, enabled: true, @@ -1462,6 +1917,7 @@ mod tests { owner_pubkey: vec![], channel_id: None, definition: serde_json::json!({}), + definition_event_id: None, definition_hash: vec![], status: status.clone(), enabled: true, @@ -1482,6 +1938,7 @@ mod tests { owner_pubkey: vec![], channel_id: None, definition: serde_json::json!({}), + definition_event_id: None, definition_hash: vec![], status: WorkflowStatus::Active, enabled: false, @@ -1774,7 +2231,7 @@ mod tests { use crate::user::ensure_user; - const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 -- local test-only credentials async fn setup_pool() -> PgPool { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index bbfcd8ecfe8..5fa70794872 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -2062,11 +2062,16 @@ pub async fn workflow_webhook( }; // Build trigger context from webhook body fields. + let Some(definition_event_id) = workflow.definition_event_id.as_deref() else { + return Err(not_found("workflow not found")); + }; let mut trigger_ctx = buzz_workflow::executor::TriggerContext { channel_id: workflow .channel_id .map(|ch| ch.to_string()) .unwrap_or_default(), + definition_event_id: hex::encode(definition_event_id), + cause: Some(buzz_workflow::executor::WorkflowCause::Webhook), ..Default::default() }; if let Some(Value::Object(ref map)) = body_json { diff --git a/crates/buzz-relay/src/api/workflows.rs b/crates/buzz-relay/src/api/workflows.rs index a3d5a6c729e..8e8ed1001f1 100644 --- a/crates/buzz-relay/src/api/workflows.rs +++ b/crates/buzz-relay/src/api/workflows.rs @@ -24,6 +24,10 @@ use crate::{ const DEFAULT_RUN_LIMIT: i64 = 20; const MAX_RUN_LIMIT: i64 = 100; +use buzz_core::workflow_delivery::{ + DEFAULT_LEASE_SECONDS as DEFAULT_DELIVERY_LEASE_SECONDS, + MAX_LEASE_SECONDS as MAX_DELIVERY_LEASE_SECONDS, +}; /// Pagination query for workflow run history. #[derive(Debug, Deserialize, Default)] @@ -195,6 +199,254 @@ pub async fn run_approvals( }))) } +/// Authenticated request to claim either a specific or the oldest due delivery. +#[derive(Debug, Deserialize)] +pub struct ClaimDeliveryRequest { + #[serde(default)] + delivery_id: Option, + #[serde(default)] + expected: Option, + #[serde(default = "default_delivery_lease_seconds")] + lease_seconds: i64, +} + +/// Immutable delivery bindings authenticated from a relay-authored live wake. +#[derive(Debug, Deserialize)] +pub struct ClaimDeliveryBindingRequest { + run_id: Uuid, + step_id: String, + definition_event_id: String, + message_event_id: String, + channel_id: Uuid, +} + +fn default_delivery_lease_seconds() -> i64 { + DEFAULT_DELIVERY_LEASE_SECONDS +} + +/// Authenticated request to extend a fenced delivery lease. +#[derive(Debug, Deserialize)] +pub struct RenewDeliveryRequest { + claim_token: Uuid, + lease_seconds: i64, +} + +/// Authenticated completion result for a fenced delivery lease. +#[derive(Debug, Deserialize)] +pub struct FinishDeliveryRequest { + claim_token: Uuid, + delivered: bool, + #[serde(default)] + retryable: bool, + #[serde(default)] + failure_code: Option, + #[serde(default)] + failure_message: Option, +} + +async fn authorize_delivery_write( + state: &Arc, + headers: &HeaderMap, + path: &str, + body: &[u8], +) -> Result<(TenantContext, nostr::PublicKey), (StatusCode, Json)> { + let raw_host = headers + .get(axum::http::header::HOST) + .and_then(|value| value.to_str().ok()) + .unwrap_or(""); + let tenant = crate::tenant::bind_community(&state.db, raw_host) + .await + .map_err(|_| api_error(StatusCode::NOT_FOUND, "relay community not found"))?; + let url = bridge::nip98_expected_url(&state.config.relay_url, &tenant, path); + let (pubkey, event_id) = bridge::verify_bridge_auth( + headers, + "POST", + &url, + Some(body), + state.config.require_auth_token, + )?; + bridge::enforce_http_admission(state, &tenant, &pubkey).await?; + bridge::check_nip98_replay(state, &tenant, event_id).await?; + let auth_tag = headers + .get("x-auth-tag") + .and_then(|value| value.to_str().ok()); + super::relay_members::enforce_relay_membership( + state, + tenant.community(), + &pubkey.to_bytes(), + auth_tag, + ) + .await?; + Ok((tenant, pubkey)) +} + +/// Claim the oldest due workflow delivery for the authenticated managed agent. +pub async fn claim_agent_delivery( + State(state): State>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result, (StatusCode, Json)> { + let path = "/workflows/agent-deliveries/claim"; + let (tenant, agent) = authorize_delivery_write(&state, &headers, path, &body).await?; + let request: ClaimDeliveryRequest = serde_json::from_slice(&body).map_err(|error| { + api_error( + StatusCode::BAD_REQUEST, + &format!("invalid claim JSON: {error}"), + ) + })?; + if !(DEFAULT_DELIVERY_LEASE_SECONDS..=MAX_DELIVERY_LEASE_SECONDS) + .contains(&request.lease_seconds) + { + return Err(api_error( + StatusCode::BAD_REQUEST, + "lease_seconds is outside the supported range", + )); + } + if request.delivery_id.is_none() && request.expected.is_some() { + return Err(api_error( + StatusCode::BAD_REQUEST, + "expected wake bindings require a specific delivery_id", + )); + } + let expected = request + .expected + .map(|binding| { + let definition_event_id = hex::decode(&binding.definition_event_id) + .map_err(|_| api_error(StatusCode::BAD_REQUEST, "invalid definition_event_id"))?; + let message_event_id = hex::decode(&binding.message_event_id) + .map_err(|_| api_error(StatusCode::BAD_REQUEST, "invalid message_event_id"))?; + if definition_event_id.len() != 32 + || message_event_id.len() != 32 + || binding.step_id.is_empty() + { + return Err(api_error( + StatusCode::BAD_REQUEST, + "invalid expected wake bindings", + )); + } + Ok(buzz_db::workflow::WorkflowAgentDeliveryBinding { + run_id: binding.run_id, + step_id: binding.step_id, + definition_event_id, + message_event_id, + channel_id: binding.channel_id, + }) + }) + .transpose()?; + let delivery = state + .db + .claim_workflow_agent_delivery( + tenant.community(), + &agent.to_bytes(), + request.delivery_id, + expected.as_ref(), + request.lease_seconds, + ) + .await + .map_err(|error| internal_error(&format!("claim workflow delivery: {error}")))?; + Ok(Json(serde_json::json!({ + "delivery": delivery.as_ref().map(delivery_json), + }))) +} + +/// Extend a workflow delivery lease using its owner/token fencing stamp. +pub async fn renew_agent_delivery( + State(state): State>, + Path(delivery_id): Path, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result, (StatusCode, Json)> { + let path = format!("/workflows/agent-deliveries/{delivery_id}/renew"); + let (tenant, agent) = authorize_delivery_write(&state, &headers, &path, &body).await?; + let request: RenewDeliveryRequest = serde_json::from_slice(&body).map_err(|error| { + api_error( + StatusCode::BAD_REQUEST, + &format!("invalid renewal JSON: {error}"), + ) + })?; + if !(DEFAULT_DELIVERY_LEASE_SECONDS..=MAX_DELIVERY_LEASE_SECONDS) + .contains(&request.lease_seconds) + { + return Err(api_error( + StatusCode::BAD_REQUEST, + "lease_seconds is outside the supported range", + )); + } + let claim_expires_at = state + .db + .renew_workflow_agent_delivery( + tenant.community(), + delivery_id, + &agent.to_bytes(), + request.claim_token, + request.lease_seconds, + ) + .await + .map_err(|error| internal_error(&format!("renew workflow delivery: {error}")))? + .ok_or_else(|| api_error(StatusCode::CONFLICT, "delivery claim is stale or expired"))?; + Ok(Json(serde_json::json!({ + "renewed": true, + "claim_expires_at": claim_expires_at, + }))) +} + +/// Complete a workflow delivery using its lease token as a fencing stamp. +pub async fn finish_agent_delivery( + State(state): State>, + Path(delivery_id): Path, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result, (StatusCode, Json)> { + let path = format!("/workflows/agent-deliveries/{delivery_id}/finish"); + let (tenant, agent) = authorize_delivery_write(&state, &headers, &path, &body).await?; + let request: FinishDeliveryRequest = serde_json::from_slice(&body).map_err(|error| { + api_error( + StatusCode::BAD_REQUEST, + &format!("invalid finish JSON: {error}"), + ) + })?; + let completed = state + .db + .finish_workflow_agent_delivery( + tenant.community(), + delivery_id, + &agent.to_bytes(), + request.claim_token, + request.delivered, + request.retryable, + request.failure_code.as_deref(), + request.failure_message.as_deref(), + ) + .await + .map_err(|error| internal_error(&format!("finish workflow delivery: {error}")))?; + if !completed { + return Err(api_error( + StatusCode::CONFLICT, + "delivery claim is stale or expired", + )); + } + Ok(Json(serde_json::json!({"completed": true}))) +} + +fn delivery_json(delivery: &buzz_db::workflow::WorkflowAgentDeliveryRecord) -> Value { + serde_json::json!({ + "id": delivery.id, + "workflow_id": delivery.workflow_id, + "run_id": delivery.run_id, + "step_id": delivery.step_id, + "definition_event_id": hex::encode(&delivery.definition_event_id), + "message_event_id": hex::encode(&delivery.message_event_id), + "channel_id": delivery.channel_id, + "target_pubkey": hex::encode(&delivery.target_pubkey), + "attempt": delivery.attempt, + "claim_token": delivery.claim_token, + "claim_expires_at": delivery.claim_expires_at, + "expires_at": delivery.expires_at, + "execution_trace": delivery.execution_trace, + "trigger_context": delivery.trigger_context, + }) +} + fn run_json(run: &buzz_db::workflow::WorkflowRunRecord) -> Value { serde_json::json!({ "id": run.id, diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 037c6b1dd3d..90c4daabed2 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -204,6 +204,15 @@ pub struct Config { /// skipped — a typo must not silently disable an operator. pub relay_operator_pubkeys: Vec, + /// Enables durable workflow-to-agent delivery (`BUZZ_WORKFLOW_AGENT_DELIVERY_ENABLED`). + /// + /// Defaults to `false` so deploying a new relay before every ACP harness has + /// durable-wake support cannot expose relay-authored workflow prompts to + /// legacy `respond-to=anyone` consumers. Operators enable this only after + /// upgrading ACP harnesses; the relay then publishes the visible kind:9 and + /// its durable delivery rows as one protocol. + pub workflow_agent_delivery_enabled: bool, + /// Allow NIP-OA owner attestation for relay membership. /// /// When `true` and `require_relay_membership` is also `true`, agents @@ -624,6 +633,9 @@ impl Config { .map(|v| v.eq_ignore_ascii_case("on") || v == "true" || v == "1") .unwrap_or(false); + let workflow_agent_delivery_enabled = + parse_bool("BUZZ_WORKFLOW_AGENT_DELIVERY_ENABLED", false)?; + let allow_nip_oa_auth = std::env::var("BUZZ_ALLOW_NIP_OA_AUTH") .map(|v| v == "true" || v == "1") .unwrap_or(false); @@ -1018,6 +1030,7 @@ impl Config { relay_owner_pubkey, relay_operator_api_origin, relay_operator_pubkeys, + workflow_agent_delivery_enabled, allow_nip_oa_auth, media, media_max_concurrent_uploads, @@ -1136,6 +1149,10 @@ mod tests { config.relay_operator_pubkeys.is_empty(), "relay_operator_pubkeys should default empty (provisioning disabled)" ); + assert!( + !config.workflow_agent_delivery_enabled, + "workflow_agent_delivery_enabled should default to false" + ); assert!( !config.allow_nip_oa_auth, "allow_nip_oa_auth should default to false" diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index d8569a7a86d..d311a98973e 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -100,6 +100,15 @@ enum PersistResult { /// operations (open_dm, hide_dm, update_approval, upsert_workflow). #[datastore_span(name = "persist_command_event", system = "postgresql")] async fn persist_command_event( + state: &Arc, + tenant: &TenantContext, + event: &Event, + channel_id_override: Option, +) -> Result { + persist_command_event_for_db(&state.db, tenant, event, channel_id_override).await +} + +async fn persist_command_event_for_db( db: &buzz_db::Db, tenant: &TenantContext, event: &Event, @@ -406,7 +415,7 @@ async fn handle_dm_open( } // Persist the command event (idempotency) — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -567,7 +576,7 @@ async fn handle_dm_add_member( } // Persist the command event — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -673,7 +682,7 @@ async fn handle_dm_hide( } // Persist the command event — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -810,7 +819,7 @@ async fn handle_workflow_def( let hash = compute_definition_hash(&definition_json_final); // Persist the command event — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -849,6 +858,8 @@ async fn handle_workflow_def( &workflow_name, &definition_json_final, &hash, + event.id.as_bytes(), + def.enabled, ) .await .map_err(|e| match e { @@ -884,6 +895,23 @@ async fn handle_workflow_def( }) } +async fn caller_controls_workflow( + state: &Arc, + community_id: CommunityId, + workflow_owner: &[u8], + caller: &[u8], +) -> Result { + if workflow_owner == caller { + return Ok(true); + } + + state + .db + .is_agent_owner(community_id, workflow_owner, caller) + .await + .map_err(|e| IngestError::Internal(format!("error: workflow owner check: {e}"))) +} + async fn handle_workflow_trigger( tenant: &TenantContext, state: &Arc, @@ -912,10 +940,11 @@ async fn handle_workflow_trigger( .await .map_err(|_| IngestError::Rejected("invalid: workflow not found".into()))?; - // 3. Manual triggers execute with the workflow owner's authority, so only - // the owner may start them. Channel membership alone is insufficient: a + // 3. Manual triggers execute with the workflow owner's authority. Permit + // that principal and, when the owner is a managed agent, its immutable + // NIP-OA human owner. Channel membership alone remains insufficient: a // member could otherwise invoke another user's webhook or message actions. - if workflow.owner_pubkey != self_bytes { + if !caller_controls_workflow(state, community_id, &workflow.owner_pubkey, &self_bytes).await? { return Err(IngestError::Rejected( "forbidden: not authorized to trigger this workflow".into(), )); @@ -949,7 +978,7 @@ async fn handle_workflow_trigger( // Persist the command event under the workflow channel even though the // trigger event itself only carries the workflow UUID. Storing channel // triggers as global events leaks workflow IDs to unrelated relay members. - let tx = match persist_command_event(&state.db, tenant, event, workflow.channel_id).await? { + let tx = match persist_command_event(state, tenant, event, workflow.channel_id).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -961,25 +990,26 @@ async fn handle_workflow_trigger( }; // 4. Execute: create workflow run - let mut trigger_ctx = TriggerContext { + let Some(definition_event_id) = workflow.definition_event_id.as_deref() else { + return Err(IngestError::Rejected( + "invalid: owner-signed workflow revision is unavailable".into(), + )); + }; + // Manual commands are signed causes, not webhook cargo. Command content is + // never copied into arbitrary trigger fields; adding parameterized manual + // runs requires an explicit trust-labelled contract. + let trigger_ctx = TriggerContext { channel_id: workflow .channel_id .map(|id| id.to_string()) .unwrap_or_default(), author: hex::encode(&self_bytes), + definition_event_id: hex::encode(definition_event_id), + cause: Some(buzz_workflow::executor::WorkflowCause::Command( + event.id.to_hex(), + )), ..Default::default() }; - if !event.content.is_empty() { - if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(&event.content) { - for (k, v) in map { - let val_str = match v { - serde_json::Value::String(s) => s, - other => other.to_string(), - }; - trigger_ctx.webhook_fields.insert(k, val_str); - } - } - } let trigger_ctx_json = serde_json::to_value(&trigger_ctx).ok(); let event_id_bytes = event.id.as_bytes().to_vec(); @@ -1133,7 +1163,7 @@ async fn handle_approval_grant( check_approver_spec(&approval.approver_spec, &self_hex)?; // Persist the command event — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -1244,7 +1274,7 @@ async fn handle_approval_deny( check_approver_spec(&approval.approver_spec, &self_hex)?; // Persist the command event — returns open transaction - let tx = match persist_command_event(&state.db, tenant, event, None).await? { + let tx = match persist_command_event(state, tenant, event, None).await? { PersistResult::Duplicate => { return Ok(IngestResult { event_id: event.id.to_hex(), @@ -1446,7 +1476,7 @@ mod tests { async fn persistence_test_context() -> (buzz_db::Db, TenantContext) { let url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 -- local test-only credentials let pool = sqlx::PgPool::connect(&url) .await .expect("connect workflow persistence test database"); @@ -1567,7 +1597,7 @@ mod tests { let created_at = Timestamp::now().as_secs(); let create = workflow_event(&keys, workflow_id, created_at, None, "create"); - let PersistResult::Inserted(tx) = persist_command_event(&db, &tenant, &create, None) + let PersistResult::Inserted(tx) = persist_command_event_for_db(&db, &tenant, &create, None) .await .expect("persist create") else { @@ -1575,7 +1605,7 @@ mod tests { }; tx.commit().await.expect("commit create"); assert!(matches!( - persist_command_event(&db, &tenant, &create, None) + persist_command_event_for_db(&db, &tenant, &create, None) .await .expect("replay create"), PersistResult::Duplicate @@ -1607,7 +1637,7 @@ mod tests { .find(|candidate| candidate.id.as_bytes() > update.id.as_bytes()) .expect("find same-second CAS-matching update dominated by current head"); - let PersistResult::Inserted(tx) = persist_command_event(&db, &tenant, &update, None) + let PersistResult::Inserted(tx) = persist_command_event_for_db(&db, &tenant, &update, None) .await .expect("persist update") else { @@ -1615,13 +1645,14 @@ mod tests { }; tx.commit().await.expect("commit update"); assert!(matches!( - persist_command_event(&db, &tenant, &update, None) + persist_command_event_for_db(&db, &tenant, &update, None) .await .expect("replay update"), PersistResult::Duplicate )); - let error = match persist_command_event(&db, &tenant, &dominated_update, None).await { + let error = match persist_command_event_for_db(&db, &tenant, &dominated_update, None).await + { Err(error) => error, Ok(_) => panic!("distinct dominated CAS update must not report duplicate success"), }; diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index ccba40f3282..dbf07f07a65 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -677,6 +677,16 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc anyhow::Result<()> { let wf_cron = Arc::clone(&workflow_engine); tokio::spawn(async move { wf_cron.run().await }); + // Durable managed-agent delivery reaper. Expired leases become retryable by + // the claim query; exhausted or TTL-expired rows become terminal here and + // emit one owner-visible channel signal from the rows returned by the + // guarded UPDATE. + { + let delivery_reaper_state = Arc::clone(&state); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + let terminal = match delivery_reaper_state + .db + .reap_workflow_agent_deliveries() + .await + { + Ok(deliveries) => deliveries, + Err(error) => { + error!(%error, "Workflow agent delivery reaper failed"); + continue; + } + }; + for delivery in terminal { + let Ok(Some(host)) = delivery_reaper_state + .db + .lookup_community_host(delivery.community_id) + .await + else { + error!(delivery_id = %delivery.id, "Workflow delivery tenant lookup failed"); + continue; + }; + let tenant = + buzz_core::tenant::TenantContext::resolved(delivery.community_id, host); + if let Err(error) = buzz_relay::handlers::side_effects::emit_system_message( + &tenant, + &delivery_reaper_state, + delivery.channel_id, + serde_json::json!({ + "type": "workflow_agent_delivery_failed", + "workflow_id": delivery.workflow_id, + "run_id": delivery.run_id, + "step_id": delivery.step_id, + "delivery_id": delivery.id, + "status": delivery.status, + }), + ) + .await + { + error!(delivery_id = %delivery.id, %error, "Workflow delivery failure signal failed"); + } + } + } + }); + } + // Ephemeral channel reaper — archives channels whose TTL deadline has passed. // Runs every 60s, matching the workflow cron loop pattern. The SQL UPDATE // uses `archived_at IS NULL` as a guard, so concurrent runs from multiple diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 1dce66e91e4..1f076474fbe 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -72,6 +72,18 @@ pub fn build_router(state: Arc) -> Router { .route("/events", post(api::bridge::submit_event)) .route("/query", post(api::bridge::query_events)) .route("/count", post(api::bridge::count_events)) + .route( + "/workflows/agent-deliveries/claim", + post(api::workflows::claim_agent_delivery), + ) + .route( + "/workflows/agent-deliveries/{delivery_id}/renew", + post(api::workflows::renew_agent_delivery), + ) + .route( + "/workflows/agent-deliveries/{delivery_id}/finish", + post(api::workflows::finish_agent_delivery), + ) .route( "/workflows/{workflow_id}/runs", get(api::workflows::workflow_runs), diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 8ce23a2e8ea..cb750eb0059 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -8,15 +8,18 @@ use std::future::Future; use std::pin::Pin; use std::sync::{Arc, Weak}; -use buzz_core::kind::KIND_STREAM_MESSAGE; +use buzz_core::kind::{KIND_STREAM_MESSAGE, KIND_WORKFLOW_AGENT_WAKE, KIND_WORKFLOW_DEF}; use buzz_core::tenant::CommunityId; -use buzz_workflow::action_sink::{ActionSink, ActionSinkError}; +use buzz_db::event::EventQuery; +use buzz_pubsub::EventTopic; +use buzz_workflow::action_sink::{ActionSink, ActionSinkError, DoorbellContext}; +use buzz_workflow::executor::WorkflowCause; use chrono::Utc; use nostr::{EventBuilder, Kind, Tag}; use tracing::info; use uuid::Uuid; -use crate::handlers::event::dispatch_persistent_event; +use crate::handlers::event::{dispatch_persistent_event, fan_out_event_to_local_subscribers}; use crate::state::AppState; /// Resolves `@Name` mentions in workflow message text to the pubkeys of the @@ -148,6 +151,17 @@ fn resolve_mention_pubkeys(text: &str, members: &[(String, String)]) -> Vec Result<(), ActionSinkError> { + if enabled { + Ok(()) + } else { + Err(ActionSinkError::InvalidInput( + "workflow agent delivery is disabled until ACP harnesses support durable wakes".into(), + )) + } +} + /// Relay-side action sink — executes workflow side-effects directly. /// /// Holds a **weak** reference to `AppState` to avoid an `Arc` reference cycle: @@ -170,17 +184,23 @@ impl RelayActionSink { } impl ActionSink for RelayActionSink { + #[allow(clippy::too_many_arguments)] fn send_message( &self, community_id: CommunityId, + workflow_id: Uuid, + step_id: &str, channel_id: &str, text: &str, author_pubkey: &str, + doorbell: &DoorbellContext, reply_to: Option<&str>, ) -> Pin> + Send + '_>> { + let step_id = step_id.to_owned(); let channel_id = channel_id.to_owned(); let text = text.to_owned(); let author_pubkey = author_pubkey.to_owned(); + let doorbell = doorbell.clone(); let reply_to = reply_to.map(str::to_owned); Box::pin(async move { @@ -190,6 +210,13 @@ impl ActionSink for RelayActionSink { .upgrade() .ok_or_else(|| ActionSinkError::Database("relay is shutting down".into()))?; + // This producer-side rollout fence is intentionally before all + // workflow-message side effects. Legacy ACPs cannot recognize the + // durable protocol marker and `respond-to=anyone` would execute a + // visible relay-authored kind:9 without claiming its delivery row. + // Operators enable the protocol only after upgrading ACP harnesses. + ensure_workflow_agent_delivery_enabled(state.config.workflow_agent_delivery_enabled)?; + // The run carries its owning community (`community_id`); the // relay-signed kind:9 message belongs to *that* community, never the // deployment default. Re-deriving the tenant from `config.relay_url` @@ -242,6 +269,58 @@ impl ActionSink for RelayActionSink { })?; let author_pubkey_bytes = author_pubkey.to_bytes().to_vec(); let author_pubkey_hex = author_pubkey.to_hex(); + // The referenced kind:30620 event is the managed agent's signed + // authority artifact. Resolve that immutable revision by exact ID; + // a human-owner management event may be the current coordinate + // replacement, but it must never become execution authority. + let mut definition_query = EventQuery::for_community(tenant.community()); + definition_query.channel_id = Some(channel_uuid); + definition_query.kinds = Some(vec![KIND_WORKFLOW_DEF as i32]); + definition_query.ids = Some(vec![hex::decode(&doorbell.definition_event_id).map_err( + |_| ActionSinkError::InvalidInput("invalid workflow definition event id".into()), + )?]); + let definition = state + .db + .query_events(&definition_query) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))? + .into_iter() + .next() + .ok_or_else(|| { + ActionSinkError::Database(format!( + "owner-signed workflow definition {workflow_id} is unavailable" + )) + })?; + let definition_event_id = definition.event.id.to_hex(); + if !definition_event_id.eq_ignore_ascii_case(&doorbell.definition_event_id) { + return Err(ActionSinkError::Database( + "workflow definition changed while the run was executing".into(), + )); + } + // Mention routing comes only from the owner-signed template, never + // from values rendered out of a trigger event or webhook payload. + // This preserves explicit workflow targets without allowing source + // text such as `{{trigger.text}}` to wake a different agent. + let (signed_workflow, _) = + buzz_workflow::WorkflowEngine::parse_yaml(&definition.event.content).map_err( + |e| ActionSinkError::InvalidInput(format!("invalid workflow definition: {e}")), + )?; + let routing_text = signed_workflow + .steps + .iter() + .find(|step| step.id == step_id) + .and_then(|step| match &step.action { + buzz_workflow::schema::ActionDef::SendMessage { text, .. } => { + Some(text.clone()) + } + _ => None, + }) + .ok_or_else(|| { + ActionSinkError::InvalidInput(format!( + "workflow step {step_id} is not a send_message action" + )) + })?; + let is_member = state .is_member_cached(tenant.community(), channel_uuid, &author_pubkey_bytes) .await @@ -254,23 +333,19 @@ impl ActionSink for RelayActionSink { // 3. Build kind:9 Nostr event // - Signed by relay keypair (event.pubkey = relay pubkey) - // - `p` tag attributes the message to the workflow owner + // - `workflow-owner` identifies the claimed principal, but ACP + // grants authority only from the referenced owner-signed definition + // - `workflow-definition` binds the exact kind:30620 event and step + // - the owner is always p-tagged; extra wake targets are resolved + // only from the owner-signed template, never rendered cause data // - `h` tag scopes to the channel (NIP-29, canonical UUID) // - `buzz:workflow` tag prevents recursive workflow triggering - // - one `p` tag per `@Name` that resolves to a channel member, - // so mentioned agents are woken (wake is `p`-tag gated) - let mut tags = vec![ - Tag::parse(["p", &author_pubkey_hex]) - .map_err(|e| ActionSinkError::EventBuild(format!("p tag: {e}")))?, - Tag::parse(["h", &channel_id_canonical]) - .map_err(|e| ActionSinkError::EventBuild(format!("h tag: {e}")))?, - Tag::parse(["buzz:workflow", "true"]) - .map_err(|e| ActionSinkError::EventBuild(format!("workflow tag: {e}")))?, - ]; - - // Resolve thread ancestry when this is a threaded reply, so the - // built event carries NIP-10 `root`/`reply` e-tags and persists real - // thread metadata (matching the ingest path) instead of top-level. + let cause_tag = match &doorbell.cause { + WorkflowCause::Event(id) => ["workflow-cause", "event", id.as_str()], + WorkflowCause::Schedule(slot) => ["workflow-cause", "schedule", slot.as_str()], + WorkflowCause::Command(id) => ["workflow-cause", "command", id.as_str()], + WorkflowCause::Webhook => ["workflow-cause", "webhook", ""], + }; let reply_ancestry = match reply_to.as_deref() { Some(parent_hex) => Some( crate::handlers::ingest::resolve_relay_reply_thread_meta( @@ -285,11 +360,47 @@ impl ActionSink for RelayActionSink { None => None, }; - // NIP-10 e-tags for the thread. Marked `root`/`reply` so clients and - // the ingest resolver read the ancestry the same way. A direct reply - // (parent == root) emits a single `reply` tag; a nested reply emits - // the `root` + `reply` pair — matching `buzz_sdk::builders::thread_tags` - // so every writer produces one wire shape per reply kind. + let mut tags = vec![ + Tag::parse(["workflow-run", doorbell.run_id.to_string().as_str()]) + .map_err(|e| ActionSinkError::EventBuild(format!("workflow-run tag: {e}")))?, + Tag::parse(["workflow-step", step_id.as_str()]) + .map_err(|e| ActionSinkError::EventBuild(format!("workflow-step tag: {e}")))?, + Tag::parse(["workflow-owner", &author_pubkey_hex]) + .map_err(|e| ActionSinkError::EventBuild(format!("workflow-owner tag: {e}")))?, + Tag::parse(["workflow-definition", &definition_event_id, &step_id]).map_err( + |e| ActionSinkError::EventBuild(format!("workflow-definition tag: {e}")), + )?, + Tag::parse(cause_tag) + .map_err(|e| ActionSinkError::EventBuild(format!("workflow-cause tag: {e}")))?, + Tag::parse(["p", &author_pubkey_hex]) + .map_err(|e| ActionSinkError::EventBuild(format!("p tag: {e}")))?, + Tag::parse(["h", &channel_id_canonical]) + .map_err(|e| ActionSinkError::EventBuild(format!("h tag: {e}")))?, + Tag::parse(["buzz:workflow", "message-v1"]) + .map_err(|e| ActionSinkError::EventBuild(format!("workflow tag: {e}")))?, + ]; + + // Resolve only owner-signed `@Name` mentions to member pubkeys. + // Dynamic trigger/webhook values are deliberately excluded from + // routing, even though ACP may render them into the local prompt. + let members = state + .db + .get_members(tenant.community(), channel_uuid) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))?; + let member_pubkeys: Vec> = members.iter().map(|m| m.pubkey.clone()).collect(); + let users = state + .db + .get_users_bulk(tenant.community(), &member_pubkeys) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))?; + let named_members: Vec<(String, String)> = users + .into_iter() + .filter_map(|u| { + let name = u.display_name?; + Some((name, nostr::PublicKey::from_slice(&u.pubkey).ok()?.to_hex())) + }) + .collect(); if let Some(ancestry) = &reply_ancestry { let root_hex = ancestry.root_hex(); let parent_hex = ancestry.parent_hex(); @@ -312,29 +423,8 @@ impl ActionSink for RelayActionSink { } } - // Resolve `@Name` mentions to channel-member pubkeys and append a - // `p` tag for each (skipping the author, already tagged above). A - // resolution failure must not drop the message, so log and proceed - // with the base tags. - let members = state - .db - .get_members(tenant.community(), channel_uuid) - .await - .map_err(|e| ActionSinkError::Database(e.to_string()))?; - let member_pubkeys: Vec> = members.iter().map(|m| m.pubkey.clone()).collect(); - let users = state - .db - .get_users_bulk(tenant.community(), &member_pubkeys) - .await - .map_err(|e| ActionSinkError::Database(e.to_string()))?; - let named_members: Vec<(String, String)> = users - .into_iter() - .filter_map(|u| { - let name = u.display_name?; - Some((name, nostr::PublicKey::from_slice(&u.pubkey).ok()?.to_hex())) - }) - .collect(); - for mentioned in resolve_mention_pubkeys(&text, &named_members) { + let mut wake_targets = vec![author_pubkey_hex.clone()]; + for mentioned in resolve_mention_pubkeys(&routing_text, &named_members) { if mentioned == author_pubkey_hex { continue; } @@ -342,80 +432,203 @@ impl ActionSink for RelayActionSink { Tag::parse(["p", &mentioned]) .map_err(|e| ActionSinkError::EventBuild(format!("mention p tag: {e}")))?, ); + wake_targets.push(mentioned); } - let kind = Kind::from(KIND_STREAM_MESSAGE as u16); - let event = EventBuilder::new(kind, &text) - .tags(tags) - .sign_with_keys(&state.relay_keypair) - .map_err(|e| ActionSinkError::EventBuild(format!("signing: {e}")))?; - - let event_id_hex = event.id.to_hex(); - let event_id_bytes = event.id.as_bytes().to_vec(); + // The owner is always the first durable target. Consult that + // identity before signing or publishing so a retry returns the + // canonical visible message instead of creating an orphan duplicate. + let (delivery_identity_lock, existing_message_event_id) = state + .db + .lock_workflow_agent_delivery_identity( + tenant.community(), + doorbell.run_id, + &step_id, + &author_pubkey_bytes, + ) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))?; let kind_u32 = KIND_STREAM_MESSAGE; - - let event_created_at = { - let ts = event.created_at.as_secs() as i64; - chrono::DateTime::from_timestamp(ts, 0).unwrap_or_else(Utc::now) + let (event, event_id_bytes, event_created_at) = if let Some((message_id, created_at)) = + existing_message_event_id + { + let stored = state + .db + .get_event_by_id(tenant.community(), &message_id) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))? + .ok_or_else(|| { + ActionSinkError::Database( + "workflow delivery references a missing visible event".into(), + ) + })?; + wake_targets = stored + .event + .tags + .iter() + .filter(|tag| tag.as_slice().first().map(String::as_str) == Some("p")) + .filter_map(|tag| tag.as_slice().get(1).cloned()) + .collect(); + (None, message_id, created_at) + } else { + let event = EventBuilder::new(Kind::from(KIND_STREAM_MESSAGE as u16), text.clone()) + .tags(tags) + .sign_with_keys(&state.relay_keypair) + .map_err(|e| ActionSinkError::EventBuild(format!("signing: {e}")))?; + let created_at = + chrono::DateTime::from_timestamp(event.created_at.as_secs() as i64, 0) + .unwrap_or_else(Utc::now); + let id = event.id.as_bytes().to_vec(); + (Some(event), id, created_at) }; + let event_id_hex = nostr::EventId::from_slice(&event_id_bytes) + .map(|id| id.to_hex()) + .map_err(|e| { + ActionSinkError::Database(format!( + "stored workflow delivery has invalid message id: {e}" + )) + })?; + info!(event_id = %event_id_hex, channel_id = %channel_id_canonical, author = %author_pubkey, + "Workflow SendMessage: reconciling kind {kind_u32} event and targets"); - info!( - event_id = %event_id_hex, - channel_id = %channel_id_canonical, - author = %author_pubkey, - "Workflow SendMessage: posting kind {kind_u32} event" - ); - - // 4. Persist event with thread metadata (matches REST handler path). - // Threaded replies persist the resolved parent/root/depth; a - // non-reply workflow message stays top-level (depth=0, no parent). - let thread_meta_owned = reply_ancestry.map(|ancestry| { - ancestry.into_thread_meta(event_id_bytes.clone(), event_created_at, channel_uuid) - }); - let thread_meta = Some(match &thread_meta_owned { - Some(owned) => owned.as_params(), - None => buzz_db::event::ThreadMetadataParams { - event_id: &event_id_bytes, + let run = state + .db + .get_workflow_run(tenant.community(), doorbell.run_id) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))?; + if run.workflow_id != workflow_id { + return Err(ActionSinkError::Database( + "workflow delivery run does not belong to workflow".into(), + )); + } + let expires_at = Utc::now() + + chrono::Duration::seconds(buzz_core::workflow_delivery::ROW_LIFETIME_SECONDS); + let targets = wake_targets + .into_iter() + .map(|pubkey| { + let key = nostr::PublicKey::from_hex(&pubkey).map_err(|e| { + ActionSinkError::InvalidInput(format!("invalid wake target: {e}")) + })?; + Ok(( + pubkey, + buzz_db::workflow::WorkflowAgentDeliveryTarget { + id: Uuid::new_v4(), + pubkey: key.to_bytes().to_vec(), + }, + )) + }) + .collect::, ActionSinkError>>()?; + let thread_meta_owned = match (event.as_ref(), reply_ancestry) { + (Some(_), Some(ancestry)) => Some(ancestry.into_thread_meta( + event_id_bytes.clone(), event_created_at, - channel_id: channel_uuid, - parent_event_id: None, - parent_event_created_at: None, - root_event_id: None, - root_event_created_at: None, - depth: 0, - broadcast: false, - }, - }); - - let (stored_event, was_inserted) = state + channel_uuid, + )), + _ => None, + }; + let top_level_thread_meta = + event + .as_ref() + .map(|_| buzz_db::event::ThreadMetadataParams { + event_id: &event_id_bytes, + event_created_at, + channel_id: channel_uuid, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: false, + }); + let thread_meta = thread_meta_owned + .as_ref() + .map(|owned| owned.as_params()) + .or(top_level_thread_meta); + let (stored_event, created_ids) = state .db - .insert_event_with_thread_metadata( + .commit_workflow_agent_deliveries( + delivery_identity_lock, tenant.community(), - &event, - Some(channel_uuid), + event.as_ref(), + &event_id_bytes, + event_created_at, thread_meta, + workflow_id, + doorbell.run_id, + &step_id, + definition.event.id.as_bytes(), + channel_uuid, + &targets + .iter() + .map( + |(_, target)| buzz_db::workflow::WorkflowAgentDeliveryTarget { + id: target.id, + pubkey: target.pubkey.clone(), + }, + ) + .collect::>(), + &run.execution_trace, + run.trigger_context.as_ref(), + expires_at, ) .await .map_err(|e| ActionSinkError::Database(e.to_string()))?; + let created_ids = created_ids + .into_iter() + .collect::>(); + for (target, delivery) in targets { + let delivery_id = delivery.id; + if !created_ids.contains(&delivery_id) { + continue; + } + + let wake = EventBuilder::new(Kind::from(KIND_WORKFLOW_AGENT_WAKE as u16), "") + .tags([ + Tag::parse(["p", target.as_str()]) + .map_err(|e| ActionSinkError::EventBuild(format!("wake p tag: {e}")))?, + Tag::parse(["h", channel_id_canonical.as_str()]) + .map_err(|e| ActionSinkError::EventBuild(format!("wake h tag: {e}")))?, + Tag::parse(["delivery", delivery_id.to_string().as_str()]).map_err( + |e| ActionSinkError::EventBuild(format!("wake delivery tag: {e}")), + )?, + Tag::parse(["workflow-definition", definition_event_id.as_str()]).map_err( + |e| ActionSinkError::EventBuild(format!("wake definition tag: {e}")), + )?, + Tag::parse(["workflow-run", doorbell.run_id.to_string().as_str()]) + .map_err(|e| { + ActionSinkError::EventBuild(format!("wake run tag: {e}")) + })?, + Tag::parse(["workflow-step", step_id.as_str()]).map_err(|e| { + ActionSinkError::EventBuild(format!("wake step tag: {e}")) + })?, + Tag::parse(["message", event_id_hex.as_str()]).map_err(|e| { + ActionSinkError::EventBuild(format!("wake message tag: {e}")) + })?, + ]) + .sign_with_keys(&state.relay_keypair) + .map_err(|e| ActionSinkError::EventBuild(format!("wake signing: {e}")))?; + state.mark_local_event(tenant.community(), &wake.id); + state + .pubsub + .publish_event(&tenant, EventTopic::Channel(channel_uuid), &wake) + .await + .map_err(|e| ActionSinkError::Database(e.to_string()))?; + let stored_wake = buzz_core::StoredEvent::new(wake, Some(channel_uuid)); + fan_out_event_to_local_subscribers(&state, tenant.community(), &stored_wake).await; + } - // 5. Post-persist side effects (fan-out, search, audit) - // Only if actually inserted (idempotency guard). - if was_inserted { + // Post-commit side effects only for the newly inserted canonical event. + if let Some(stored_event) = stored_event.as_ref() { let _ = dispatch_persistent_event( &tenant, &state, - &stored_event, + stored_event, kind_u32, &author_pubkey_hex, None, ) .await; - // A threaded reply changed its thread's counters — push a fresh - // relay-signed kind:39005 so subscribed clients update badge - // counts without refetching the head window, exactly as the - // ingest path does after a reply insert. Fan-out-only and - // best-effort; skipped for top-level (non-reply) messages. if let Some(owned) = &thread_meta_owned { crate::handlers::side_effects::emit_live_thread_summary( &tenant, @@ -557,6 +770,17 @@ mod tests { // dropped as vacuous — `ẞ` lowercases to `ß`, one char, so it never inverts // original-vs-folded length; only `İ` does). + #[test] + fn producer_gate_blocks_legacy_anyone_mixed_version_rollout() { + let error = ensure_workflow_agent_delivery_enabled(false) + .expect_err("disabled producer must not emit a visible kind:9"); + assert!(matches!(error, ActionSinkError::InvalidInput(_))); + assert!( + ensure_workflow_agent_delivery_enabled(true).is_ok(), + "operator may enable delivery after ACP harnesses are upgraded" + ); + } + #[test] fn combining_mark_in_name_matches() { // A name carrying a combining mark (`é` as `e` + U+0301) matches the same @@ -626,503 +850,5 @@ mod tests { } #[cfg(test)] -mod integration_tests { - //! Regression test for `e3661764` / `7899c1a8`: a workflow `send_message` - //! that mentions a channel member by name (`@Name`) must emit a `p` tag for - //! that member so ACP agent wake (`event_mentions_agent`, p-tag gated) fires. - //! - //! Postgres-gated like the other DB-backed relay tests. Run with: - //! `cargo test -p buzz-relay --lib workflow_sink -- --ignored` - use super::*; - use buzz_core::channel::{ChannelType, ChannelVisibility, MemberRole}; - use buzz_db::CreateCommunityWithOwnerResult; - use std::sync::Arc; - - /// Real-PG state mirroring `handlers::event::tests::test_state_with_redis_url`. - async fn test_state() -> Arc { - let mut config = crate::config::Config::from_env().expect("default config loads"); - config.require_relay_membership = false; - config.redis_url = "redis://127.0.0.1:1".to_string(); - let pool = sqlx::PgPool::connect_lazy(&config.database_url).expect("lazy pg pool"); - let db = buzz_db::Db::from_pool(pool.clone()); - let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) - .create_pool(Some(deadpool_redis::Runtime::Tokio1)) - .expect("redis pool"); - let pubsub = Arc::new( - buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) - .await - .expect("pubsub manager"), - ); - let audit = buzz_audit::AuditService::new(pool.clone()); - let auth = buzz_auth::AuthService::new(config.auth.clone()); - let search = buzz_search::SearchService::new(pool.clone()); - let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( - db.clone(), - buzz_workflow::WorkflowConfig::default(), - )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); - let (state, _audit_shutdown) = AppState::new( - config, - db, - redis_pool, - audit, - pubsub, - auth, - search, - workflow_engine, - nostr::Keys::generate(), - media_storage, - ); - Arc::new(state) - } - - #[tokio::test] - #[ignore = "requires Postgres"] - async fn workflow_send_message_p_tags_mentioned_member() { - let state = test_state().await; - - let author = nostr::Keys::generate(); - let author_hex = author.public_key().to_hex(); - let agent = nostr::Keys::generate(); - let agent_hex = agent.public_key().to_hex(); - let agent_bytes = agent.public_key().to_bytes().to_vec(); - - let host = format!("wf-ptag-{}.example", uuid::Uuid::new_v4().simple()); - let community = match state - .db - .create_community_with_owner(&host, &author_hex) - .await - .expect("create community") - { - CreateCommunityWithOwnerResult::Created(rec) => rec.id, - other => panic!("expected fresh community, got {other:?}"), - }; - - // Open channel; the creator (author) is bootstrapped as an owner-member. - let channel = state - .db - .create_channel( - community, - "wf-ptag", - ChannelType::Stream, - ChannelVisibility::Open, - None, - &author.public_key().to_bytes(), - None, - ) - .await - .expect("create channel"); - - // The mentioned agent is a real member with a resolvable display name. - state - .db - .ensure_user(community, &agent_bytes) - .await - .expect("ensure agent user row"); - state - .db - .update_user_profile(community, &agent_bytes, Some("Robby"), None, None, None) - .await - .expect("set agent display name"); - state - .db - .add_member( - community, - channel.id, - &agent_bytes, - MemberRole::Bot, - Some(&author.public_key().to_bytes()), - ) - .await - .expect("add agent member"); - - let sink = RelayActionSink::new(&state); - let event_id_hex = sink - .send_message( - community, - &channel.id.to_string(), - "heads up @Robby — please take a look", - &author_hex, - None, - ) - .await - .expect("send_message"); - - let id_bytes = nostr::EventId::from_hex(&event_id_hex) - .expect("event id") - .as_bytes() - .to_vec(); - let stored = state - .db - .get_event_by_id(community, &id_bytes) - .await - .expect("query event") - .expect("event persisted"); - - let p_tag_targets: Vec<&str> = stored - .event - .tags - .iter() - .filter(|t| t.as_slice().first().map(|s| s.as_str()) == Some("p")) - .filter_map(|t| t.as_slice().get(1).map(|s| s.as_str())) - .collect(); - - assert!( - p_tag_targets.contains(&author_hex.as_str()), - "author should still be attributed via p tag; got {p_tag_targets:?}" - ); - assert!( - p_tag_targets.contains(&agent_hex.as_str()), - "mentioned member {agent_hex} must be p-tagged so it wakes; got {p_tag_targets:?}" - ); - } - - #[tokio::test] - #[ignore = "requires Postgres"] - async fn workflow_reply_in_thread_threads_onto_parent() { - let state = test_state().await; - - let author = nostr::Keys::generate(); - let author_hex = author.public_key().to_hex(); - - let host = format!("wf-thread-{}.example", uuid::Uuid::new_v4().simple()); - let community = match state - .db - .create_community_with_owner(&host, &author_hex) - .await - .expect("create community") - { - CreateCommunityWithOwnerResult::Created(rec) => rec.id, - other => panic!("expected fresh community, got {other:?}"), - }; - - let channel = state - .db - .create_channel( - community, - "wf-thread", - ChannelType::Stream, - ChannelVisibility::Open, - None, - &author.public_key().to_bytes(), - None, - ) - .await - .expect("create channel"); - - let sink = RelayActionSink::new(&state); - - // 1. A top-level workflow message becomes the thread root. - let root_hex = sink - .send_message( - community, - &channel.id.to_string(), - "root message", - &author_hex, - None, - ) - .await - .expect("send root"); - - // 2. A reply_in_thread message threads onto it. - let reply_hex = sink - .send_message( - community, - &channel.id.to_string(), - "threaded reply", - &author_hex, - Some(&root_hex), - ) - .await - .expect("send reply"); - - // A direct reply carries a single NIP-10 reply e-tag at the root (no - // root marker), matching SDK `thread_tags`. - let reply_id_bytes = nostr::EventId::from_hex(&reply_hex) - .expect("reply id") - .as_bytes() - .to_vec(); - let stored = state - .db - .get_event_by_id(community, &reply_id_bytes) - .await - .expect("query reply") - .expect("reply persisted"); - let marker = |m: &str| -> Option { - stored.event.tags.iter().find_map(|t| { - let p = t.as_slice(); - if p.len() >= 4 && p[0] == "e" && p[3] == m { - Some(p[1].clone()) - } else { - None - } - }) - }; - assert_eq!( - marker("reply").as_deref(), - Some(root_hex.as_str()), - "direct reply emits a single reply marker at the root" - ); - assert_eq!( - marker("root"), - None, - "direct reply omits the root marker (matches SDK thread_tags)" - ); - - // Thread metadata reflects a depth-1 reply parented on the root. - let meta = state - .db - .get_thread_metadata_by_event(community, &reply_id_bytes) - .await - .expect("query meta") - .expect("reply has thread metadata"); - assert_eq!( - meta.depth, 1, - "direct reply to a top-level message is depth 1" - ); - let root_bytes = nostr::EventId::from_hex(&root_hex) - .expect("root id") - .as_bytes() - .to_vec(); - assert_eq!(meta.parent_event_id.as_deref(), Some(root_bytes.as_slice())); - assert_eq!(meta.root_event_id.as_deref(), Some(root_bytes.as_slice())); - } - - #[tokio::test] - #[ignore = "requires Postgres"] - async fn workflow_replies_recover_metadata_less_parent_ancestry() { - // A parent that carries NIP-10 root/reply markers but has NO - // thread_metadata row (legacy or not-yet-indexed) must be recognized as - // nested: the workflow reply threads at depth 2 onto the parent's own - // root, not a false top-level depth 1. - let state = test_state().await; - - let author = nostr::Keys::generate(); - let author_hex = author.public_key().to_hex(); - - let host = format!("wf-legacy-{}.example", uuid::Uuid::new_v4().simple()); - let community = match state - .db - .create_community_with_owner(&host, &author_hex) - .await - .expect("create community") - { - CreateCommunityWithOwnerResult::Created(rec) => rec.id, - other => panic!("expected fresh community, got {other:?}"), - }; - - let channel = state - .db - .create_channel( - community, - "wf-legacy", - ChannelType::Stream, - ChannelVisibility::Open, - None, - &author.public_key().to_bytes(), - None, - ) - .await - .expect("create channel"); - - let channel_hex = channel.id.to_string(); - - // A top-level root message, inserted WITHOUT any thread metadata row. - let root_event = EventBuilder::new(Kind::from(KIND_STREAM_MESSAGE as u16), "root") - .tags([Tag::parse(["h", &channel_hex]).expect("h tag")]) - .sign_with_keys(&author) - .expect("sign root"); - let root_hex = root_event.id.to_hex(); - state - .db - .insert_event(community, &root_event, Some(channel.id)) - .await - .expect("insert root"); - - // A nested parent that marks its root/reply — but, crucially, is stored - // with NO thread_metadata row (the legacy/unindexed case F1 addresses). - let parent_event = - EventBuilder::new(Kind::from(KIND_STREAM_MESSAGE as u16), "nested parent") - .tags([ - Tag::parse(["h", &channel_hex]).expect("h tag"), - Tag::parse(["e", &root_hex, "", "root"]).expect("root tag"), - Tag::parse(["e", &root_hex, "", "reply"]).expect("reply tag"), - ]) - .sign_with_keys(&author) - .expect("sign parent"); - let parent_hex = parent_event.id.to_hex(); - state - .db - .insert_event(community, &parent_event, Some(channel.id)) - .await - .expect("insert parent"); - assert!( - state - .db - .get_thread_metadata_by_event(community, parent_event.id.as_bytes()) - .await - .expect("query parent meta") - .is_none(), - "test premise: the nested parent must have no thread_metadata row" - ); - - // A workflow reply onto the metadata-less nested parent. - let reply_hex = RelayActionSink::new(&state) - .send_message( - community, - &channel_hex, - "workflow reply", - &author_hex, - Some(&parent_hex), - ) - .await - .expect("send reply"); - - let reply_id_bytes = nostr::EventId::from_hex(&reply_hex) - .expect("reply id") - .as_bytes() - .to_vec(); - let meta = state - .db - .get_thread_metadata_by_event(community, &reply_id_bytes) - .await - .expect("query meta") - .expect("reply has thread metadata"); - - assert_eq!( - meta.depth, 2, - "reply to a marked-but-unindexed nested parent is depth 2, not top-level" - ); - let root_bytes = nostr::EventId::from_hex(&root_hex) - .expect("root id") - .as_bytes() - .to_vec(); - let parent_bytes = parent_event.id.as_bytes().to_vec(); - assert_eq!( - meta.root_event_id.as_deref(), - Some(root_bytes.as_slice()), - "root recovered from the parent's own NIP-10 markers" - ); - assert_eq!( - meta.parent_event_id.as_deref(), - Some(parent_bytes.as_slice()) - ); - - // The reply's own NIP-10 e-tags point root→the recovered root, - // reply→the immediate parent (matching the ingest resolver). - let stored = state - .db - .get_event_by_id(community, &reply_id_bytes) - .await - .expect("query reply") - .expect("reply persisted"); - let marker = |m: &str| -> Option { - stored.event.tags.iter().find_map(|t| { - let p = t.as_slice(); - if p.len() >= 4 && p[0] == "e" && p[3] == m { - Some(p[1].clone()) - } else { - None - } - }) - }; - assert_eq!(marker("root").as_deref(), Some(root_hex.as_str())); - assert_eq!(marker("reply").as_deref(), Some(parent_hex.as_str())); - - // A root-only parent is top-level under the shared collapse rule, even - // without metadata. A workflow reply therefore starts a thread at P, - // rather than incorrectly inheriting the marker's unrelated root R. - let root_only_parent = - EventBuilder::new(Kind::from(KIND_STREAM_MESSAGE as u16), "root-only parent") - .tags([ - Tag::parse(["h", &channel_hex]).expect("h tag"), - Tag::parse(["e", &root_hex, "", "root"]).expect("root tag"), - ]) - .sign_with_keys(&author) - .expect("sign root-only parent"); - let root_only_parent_hex = root_only_parent.id.to_hex(); - let root_only_parent_bytes = root_only_parent.id.as_bytes().to_vec(); - state - .db - .insert_event(community, &root_only_parent, Some(channel.id)) - .await - .expect("insert root-only parent"); - - let root_only_reply_hex = RelayActionSink::new(&state) - .send_message( - community, - &channel_hex, - "workflow reply to root-only parent", - &author_hex, - Some(&root_only_parent_hex), - ) - .await - .expect("send root-only reply"); - let root_only_reply_bytes = nostr::EventId::from_hex(&root_only_reply_hex) - .expect("reply id") - .as_bytes() - .to_vec(); - let root_only_meta = state - .db - .get_thread_metadata_by_event(community, &root_only_reply_bytes) - .await - .expect("query root-only reply meta") - .expect("root-only reply has thread metadata"); - assert_eq!(root_only_meta.depth, 1); - assert_eq!( - root_only_meta.parent_event_id.as_deref(), - Some(root_only_parent_bytes.as_slice()) - ); - assert_eq!( - root_only_meta.root_event_id.as_deref(), - Some(root_only_parent_bytes.as_slice()) - ); - } - - #[tokio::test] - #[ignore = "requires Postgres"] - async fn workflow_reply_to_missing_parent_errors() { - let state = test_state().await; - let author = nostr::Keys::generate(); - let author_hex = author.public_key().to_hex(); - let host = format!("wf-missing-{}.example", uuid::Uuid::new_v4().simple()); - let community = match state - .db - .create_community_with_owner(&host, &author_hex) - .await - .expect("create community") - { - CreateCommunityWithOwnerResult::Created(rec) => rec.id, - other => panic!("expected fresh community, got {other:?}"), - }; - let channel = state - .db - .create_channel( - community, - "wf-missing", - ChannelType::Stream, - ChannelVisibility::Open, - None, - &author.public_key().to_bytes(), - None, - ) - .await - .expect("create channel"); - - let unknown = nostr::Keys::generate().public_key().to_hex(); - let err = RelayActionSink::new(&state) - .send_message( - community, - &channel.id.to_string(), - "orphan reply", - &author_hex, - Some(&unknown), - ) - .await - .expect_err("reply to a non-existent parent must fail"); - assert!( - matches!(err, ActionSinkError::InvalidInput(_)), - "expected InvalidInput, got {err:?}" - ); - } -} +#[path = "workflow_sink/integration_tests.rs"] +mod integration_tests; diff --git a/crates/buzz-relay/src/workflow_sink/integration_tests.rs b/crates/buzz-relay/src/workflow_sink/integration_tests.rs new file mode 100644 index 00000000000..5f725e9806b --- /dev/null +++ b/crates/buzz-relay/src/workflow_sink/integration_tests.rs @@ -0,0 +1,1202 @@ +//! Doorbell routing regression: a workflow's owner-signed `@Name` may add +//! a wake target, but dynamic trigger/webhook values cannot become routing +//! authority because mention extraction reads the signed template only. +//! +//! Postgres-gated like the other DB-backed relay tests. Run with: +//! `cargo test -p buzz-relay --lib workflow_sink -- --ignored` +use super::*; +use buzz_core::channel::{ChannelType, ChannelVisibility, MemberRole}; +use buzz_db::CreateCommunityWithOwnerResult; +use std::sync::Arc; + +/// Real-PG state mirroring `handlers::event::tests::test_state_with_redis_url`. +async fn test_state_with_database_url(database_url: Option) -> Arc { + let mut config = crate::config::Config::from_env().expect("default config loads"); + if let Some(database_url) = database_url { + config.database_url = database_url; + } + config.require_relay_membership = false; + config.workflow_agent_delivery_enabled = true; + let pool = sqlx::PgPool::connect_lazy(&config.database_url).expect("lazy pg pool"); + let db = buzz_db::Db::from_pool(pool.clone()); + let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("redis pool"); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) + .await + .expect("pubsub manager"), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let (state, _audit_shutdown) = AppState::new( + config, + db, + redis_pool, + audit, + pubsub, + auth, + search, + workflow_engine, + nostr::Keys::generate(), + media_storage, + ); + Arc::new(state) +} + +async fn test_state() -> Arc { + test_state_with_database_url(None).await +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn desired_schema_only_bootstrap_runs_durable_workflow_delivery_path() { + let base_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| { + "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string() // sadscan:disable np.postgres.1 -- local test-only credentials + }); + let admin = sqlx::PgPool::connect(&base_url) + .await + .expect("connect admin database"); + let scratch_name = format!("buzz_workflow_schema_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!( + "CREATE DATABASE {scratch_name}" + ))) + .execute(&admin) + .await + .expect("create desired-schema workflow database"); + let (base_prefix, _) = base_url.rsplit_once('/').expect("database URL has path"); + let scratch_url = format!("{base_prefix}/{scratch_name}"); + let bootstrap = sqlx::PgPool::connect(&scratch_url) + .await + .expect("connect desired-schema workflow database"); + sqlx::raw_sql(sqlx::AssertSqlSafe(include_str!( + "../../../../schema/schema.sql" + ))) + .execute(&bootstrap) + .await + .expect("apply desired-state schema without migrations"); + bootstrap.close().await; + + let runtime_pool = sqlx::PgPool::connect(&scratch_url) + .await + .expect("connect runtime assertion pool"); + let state = test_state_with_database_url(Some(scratch_url)).await; + let owner = nostr::Keys::generate(); + let owner_hex = owner.public_key().to_hex(); + let host = format!("wf-schema-{}.example", Uuid::new_v4().simple()); + let community = match state + .db + .create_community_with_owner(&host, &owner_hex) + .await + .expect("create community through runtime DB path") + { + CreateCommunityWithOwnerResult::Created(record) => record.id, + other => panic!("expected fresh community, got {other:?}"), + }; + state + .db + .ensure_user(community, &owner.public_key().to_bytes()) + .await + .expect("ensure workflow owner"); + let channel = state + .db + .create_channel( + community, + "schema-workflow", + ChannelType::Stream, + ChannelVisibility::Open, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .expect("create workflow channel"); + let workflow_id = Uuid::new_v4(); + let definition = EventBuilder::new( + Kind::Custom(KIND_WORKFLOW_DEF as u16), + "name: schema-only\ntrigger:\n on: webhook\nsteps:\n - id: notify\n action: send_message\n text: schema-only\n", + ) + .tags([ + Tag::parse(["d", &workflow_id.to_string()]).expect("d tag"), + Tag::parse(["h", &channel.id.to_string()]).expect("h tag"), + ]) + .sign_with_keys(&owner) + .expect("sign workflow definition"); + state + .db + .insert_event(community, &definition, Some(channel.id)) + .await + .expect("persist workflow definition"); + let (_, definition_json) = + buzz_workflow::WorkflowEngine::parse_yaml(&definition.content).expect("parse workflow"); + let definition_hash = + ::digest(definition_json.as_bytes()).to_vec(); + state + .db + .upsert_workflow( + community, + workflow_id, + Some(channel.id), + &owner.public_key().to_bytes(), + "schema-only", + &definition_json, + &definition_hash, + definition.id.as_bytes(), + true, + ) + .await + .expect("materialize workflow"); + let trigger = serde_json::to_value(buzz_workflow::executor::TriggerContext { + channel_id: channel.id.to_string(), + definition_event_id: definition.id.to_hex(), + cause: Some(WorkflowCause::Webhook), + ..Default::default() + }) + .expect("serialize trigger"); + let run_id = state + .db + .create_workflow_run(community, workflow_id, None, Some(&trigger)) + .await + .expect("create workflow run"); + state + .db + .update_workflow_run( + community, + run_id, + buzz_db::workflow::RunStatus::Running, + 0, + &serde_json::json!([]), + None, + ) + .await + .expect("start workflow run"); + let event_id = RelayActionSink::new(&state) + .send_message( + community, + workflow_id, + "notify", + &channel.id.to_string(), + "schema-only", + &owner_hex, + &DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }, + None, + ) + .await + .expect("run actual durable workflow message path"); + let delivery_id: Uuid = sqlx::query_scalar( + "SELECT id FROM workflow_agent_deliveries WHERE community_id = $1 AND run_id = $2", + ) + .bind(community.as_uuid()) + .bind(run_id) + .fetch_one(&runtime_pool) + .await + .expect("lookup desired-schema delivery identity"); + let wrong_binding = buzz_db::workflow::WorkflowAgentDeliveryBinding { + run_id, + step_id: "notify".to_string(), + definition_event_id: definition.id.as_bytes().to_vec(), + message_event_id: nostr::EventId::from_hex(&event_id) + .unwrap() + .as_bytes() + .to_vec(), + channel_id: Uuid::new_v4(), + }; + assert!( + state + .db + .claim_workflow_agent_delivery( + community, + &owner.public_key().to_bytes(), + Some(delivery_id), + Some(&wrong_binding), + 120, + ) + .await + .expect("reject mismatched live-wake binding before claim") + .is_none(), + "an authenticated but cross-channel wake must not mutate the victim delivery" + ); + let binding = buzz_db::workflow::WorkflowAgentDeliveryBinding { + channel_id: channel.id, + ..wrong_binding + }; + let delivery = state + .db + .claim_workflow_agent_delivery( + community, + &owner.public_key().to_bytes(), + Some(delivery_id), + Some(&binding), + 120, + ) + .await + .expect("claim desired-schema delivery") + .expect("workflow path created a durable delivery"); + assert_eq!(delivery.run_id, run_id); + assert_eq!( + delivery.message_event_id, + nostr::EventId::from_hex(&event_id).unwrap().as_bytes() + ); + + drop(state); + runtime_pool.close().await; + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {scratch_name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop desired-schema workflow database"); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn workflow_doorbell_routes_from_signed_template() { + let state = test_state().await; + let pool = sqlx::PgPool::connect(&state.config.database_url) + .await + .expect("connect integration pool"); + + let author = nostr::Keys::generate(); + let author_hex = author.public_key().to_hex(); + let agent = nostr::Keys::generate(); + let agent_bytes = agent.public_key().to_bytes().to_vec(); + + let host = format!("wf-ptag-{}.example", uuid::Uuid::new_v4().simple()); + let community = match state + .db + .create_community_with_owner(&host, &author_hex) + .await + .expect("create community") + { + CreateCommunityWithOwnerResult::Created(rec) => rec.id, + other => panic!("expected fresh community, got {other:?}"), + }; + + state + .db + .ensure_user(community, &author.public_key().to_bytes()) + .await + .expect("ensure workflow author user row"); + + // Open channel; the creator (author) is bootstrapped as an owner-member. + let channel = state + .db + .create_channel( + community, + "wf-ptag", + ChannelType::Stream, + ChannelVisibility::Open, + None, + &author.public_key().to_bytes(), + None, + ) + .await + .expect("create channel"); + + // The mentioned agent is a real member with a resolvable display name. + state + .db + .ensure_user(community, &agent_bytes) + .await + .expect("ensure agent user row"); + state + .db + .update_user_profile(community, &agent_bytes, Some("Robby"), None, None, None) + .await + .expect("set agent display name"); + state + .db + .add_member( + community, + channel.id, + &agent_bytes, + MemberRole::Bot, + Some(&author.public_key().to_bytes()), + ) + .await + .expect("add agent member"); + + let workflow_id = Uuid::new_v4(); + let definition = EventBuilder::new( + Kind::Custom(KIND_WORKFLOW_DEF as u16), + "name: ptag\ntrigger:\n on: webhook\nsteps:\n - id: notify\n action: send_message\n text: '{{trigger.text}}'\n - id: follow_up\n action: send_message\n text: 'previous={{steps.call.output.body}}'\n - id: concurrent\n action: send_message\n text: 'parallel'\n - id: atomic_targets\n action: send_message\n text: 'atomic @Robby'\n - id: threaded\n action: send_message\n text: 'threaded @Robby'\n", + ) + .tags([ + Tag::parse(["d", &workflow_id.to_string()]).expect("d tag"), + Tag::parse(["h", &channel.id.to_string()]).expect("h tag"), + ]) + .sign_with_keys(&author) + .expect("sign definition"); + state + .db + .insert_event(community, &definition, Some(channel.id)) + .await + .expect("persist definition"); + + let (_, definition_json) = + buzz_workflow::WorkflowEngine::parse_yaml(&definition.content).expect("parse definition"); + let definition_hash = + ::digest(definition_json.as_bytes()).to_vec(); + state + .db + .upsert_workflow( + community, + workflow_id, + Some(channel.id), + &author.public_key().to_bytes(), + "ptag", + &definition_json, + &definition_hash, + definition.id.as_bytes(), + true, + ) + .await + .expect("materialize workflow"); + let trigger_context = buzz_workflow::executor::TriggerContext { + channel_id: channel.id.to_string(), + definition_event_id: definition.id.to_hex(), + cause: Some(WorkflowCause::Webhook), + webhook_fields: [("private_token".to_string(), "relay-secret".to_string())].into(), + ..Default::default() + }; + let trigger_json = serde_json::to_value(&trigger_context).expect("trigger JSON"); + let execution_trace = serde_json::json!([{ + "step_id": "call", + "output": {"body": "durable-prior-output"} + }]); + let run_id = state + .db + .create_workflow_run(community, workflow_id, None, Some(&trigger_json)) + .await + .expect("create run"); + state + .db + .update_workflow_run( + community, + run_id, + buzz_db::workflow::RunStatus::Running, + 1, + &execution_trace, + None, + ) + .await + .expect("persist prior-step trace"); + + let sink = RelayActionSink::new(&state); + + // Current-main reply threading and durable delivery must commit as one slice. + let thread_root = EventBuilder::new(Kind::from(KIND_STREAM_MESSAGE as u16), "root") + .tags([Tag::parse(["h", &channel.id.to_string()]).expect("root h tag")]) + .sign_with_keys(&author) + .expect("sign thread root"); + let root_created_at = + chrono::DateTime::from_timestamp(thread_root.created_at.as_secs() as i64, 0) + .expect("valid root timestamp"); + state + .db + .insert_event_with_thread_metadata( + community, + &thread_root, + Some(channel.id), + Some(buzz_db::event::ThreadMetadataParams { + event_id: thread_root.id.as_bytes(), + event_created_at: root_created_at, + channel_id: channel.id, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: false, + }), + ) + .await + .expect("persist thread root"); + let threaded_event_id = sink + .send_message( + community, + workflow_id, + "threaded", + &channel.id.to_string(), + "threaded @Robby", + &author_hex, + &DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }, + Some(&thread_root.id.to_hex()), + ) + .await + .expect("send threaded workflow message"); + let threaded_event_bytes = nostr::EventId::from_hex(&threaded_event_id) + .expect("threaded event id") + .as_bytes() + .to_vec(); + let threaded_meta = state + .db + .get_thread_metadata_by_event(community, &threaded_event_bytes) + .await + .expect("load threaded metadata") + .expect("threaded metadata committed"); + assert_eq!( + threaded_meta.parent_event_id.as_deref(), + Some(thread_root.id.as_bytes().as_slice()) + ); + assert_eq!( + threaded_meta.root_event_id.as_deref(), + Some(thread_root.id.as_bytes().as_slice()) + ); + assert_eq!(threaded_meta.depth, 1); + let threaded_targets: i64 = sqlx::query_scalar( + "SELECT count(*) FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND step_id='threaded' AND message_event_id=$3", + ) + .bind(community.as_uuid()) + .bind(run_id) + .bind(&threaded_event_bytes) + .fetch_one(&pool) + .await + .expect("count threaded delivery targets"); + assert_eq!(threaded_targets, 2); + sqlx::query( + "UPDATE workflow_agent_deliveries SET status='delivered' WHERE community_id=$1 AND run_id=$2 AND step_id='threaded'", + ) + .bind(community.as_uuid()) + .bind(run_id) + .execute(&pool) + .await + .expect("complete threaded regression deliveries"); + + // Inject a failure after the event insert reaches the transaction but before + // any durable target can commit. The retry must leave one event and all targets. + sqlx::raw_sql( + r#" + CREATE OR REPLACE FUNCTION fail_workflow_delivery_insert() RETURNS trigger AS $$ + BEGIN RAISE EXCEPTION 'injected delivery failure'; END; $$ LANGUAGE plpgsql; + CREATE TRIGGER fail_workflow_delivery BEFORE INSERT ON workflow_agent_deliveries + FOR EACH ROW EXECUTE FUNCTION fail_workflow_delivery_insert(); + "#, + ) + .execute(&pool) + .await + .expect("install after-event failure"); + let atomic_doorbell = DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }; + assert!(sink + .send_message( + community, + workflow_id, + "atomic_targets", + &channel.id.to_string(), + "atomic @Robby", + &author_hex, + &atomic_doorbell, + None, + ) + .await + .is_err()); + sqlx::query("DROP TRIGGER fail_workflow_delivery ON workflow_agent_deliveries") + .execute(&pool) + .await + .expect("remove after-event failure"); + let atomic_event_id = sink + .send_message( + community, + workflow_id, + "atomic_targets", + &channel.id.to_string(), + "atomic @Robby", + &author_hex, + &atomic_doorbell, + None, + ) + .await + .expect("retry after event-boundary rollback"); + let atomic_event_bytes = nostr::EventId::from_hex(&atomic_event_id) + .unwrap() + .as_bytes() + .to_vec(); + let atomic_visible: i64 = + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(&atomic_event_bytes) + .fetch_one(&pool) + .await + .unwrap(); + let atomic_targets: i64 = sqlx::query_scalar( + "SELECT count(*) FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND step_id='atomic_targets'") + .bind(community.as_uuid()).bind(run_id).fetch_one(&pool).await.unwrap(); + assert_eq!((atomic_visible, atomic_targets), (1, 2)); + + // Now remove the committed atomic slice and fail specifically on the second + // (mentioned-agent) target. Owner insertion precedes it inside the same + // transaction; rollback plus retry must restore the identical all-target state. + sqlx::query("DELETE FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND step_id='atomic_targets'") + .bind(community.as_uuid()).bind(run_id).execute(&pool).await.unwrap(); + sqlx::query("DELETE FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(&atomic_event_bytes) + .execute(&pool) + .await + .unwrap(); + sqlx::raw_sql( + r#" + CREATE OR REPLACE FUNCTION fail_second_workflow_target() RETURNS trigger AS $$ + BEGIN IF encode(NEW.target_pubkey, 'hex') = current_setting('buzz.test_fail_target') THEN + RAISE EXCEPTION 'injected second target failure'; END IF; RETURN NEW; END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER fail_second_workflow_target BEFORE INSERT ON workflow_agent_deliveries + FOR EACH ROW EXECUTE FUNCTION fail_second_workflow_target(); + "#, + ) + .execute(&pool) + .await + .expect("install second-target failure"); + sqlx::query("SELECT set_config('buzz.test_fail_target', $1, false)") + .bind(agent.public_key().to_hex()) + .execute(&pool) + .await + .expect("set second failure target"); + assert!(sink + .send_message( + community, + workflow_id, + "atomic_targets", + &channel.id.to_string(), + "atomic @Robby", + &author_hex, + &atomic_doorbell, + None, + ) + .await + .is_err()); + sqlx::query("DROP TRIGGER fail_second_workflow_target ON workflow_agent_deliveries") + .execute(&pool) + .await + .expect("remove second-target failure"); + let recovered_id = sink + .send_message( + community, + workflow_id, + "atomic_targets", + &channel.id.to_string(), + "atomic @Robby", + &author_hex, + &atomic_doorbell, + None, + ) + .await + .expect("retry after second-target rollback"); + let recovered_bytes = nostr::EventId::from_hex(&recovered_id) + .unwrap() + .as_bytes() + .to_vec(); + let recovered_visible: i64 = + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(&recovered_bytes) + .fetch_one(&pool) + .await + .unwrap(); + let recovered_targets: i64 = sqlx::query_scalar( + "SELECT count(*) FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND step_id='atomic_targets' AND message_event_id=$3") + .bind(community.as_uuid()).bind(run_id).bind(&recovered_bytes).fetch_one(&pool).await.unwrap(); + assert_eq!((recovered_visible, recovered_targets), (1, 2)); + sqlx::query("UPDATE workflow_agent_deliveries SET status='delivered' WHERE community_id=$1 AND run_id=$2 AND step_id='atomic_targets'") + .bind(community.as_uuid()).bind(run_id).execute(&pool).await.unwrap(); + + let event_id_hex = sink + .send_message( + community, + workflow_id, + "notify", + &channel.id.to_string(), + "heads up @Robby — please take a look", + &author_hex, + &DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }, + None, + ) + .await + .expect("send_message"); + + let id_bytes = nostr::EventId::from_hex(&event_id_hex) + .expect("event id") + .as_bytes() + .to_vec(); + let stored = state + .db + .get_event_by_id(community, &id_bytes) + .await + .expect("query event") + .expect("event persisted"); + + let p_tag_targets: Vec<&str> = stored + .event + .tags + .iter() + .filter(|t| t.as_slice().first().map(|s| s.as_str()) == Some("p")) + .filter_map(|t| t.as_slice().get(1).map(|s| s.as_str())) + .collect(); + + assert_eq!( + p_tag_targets, + vec![author_hex.as_str()], + "rendered @Robby from trigger data must not become a second wake target" + ); + + let workflow_owner_targets: Vec<&str> = stored + .event + .tags + .iter() + .filter(|t| t.as_slice().first().map(|s| s.as_str()) == Some("workflow-owner")) + .filter_map(|t| t.as_slice().get(1).map(|s| s.as_str())) + .collect(); + assert_eq!( + workflow_owner_targets, + vec![author_hex.as_str()], + "workflow output must carry exactly one dedicated owner authority tag" + ); + + let definition_targets: Vec<&[String]> = stored + .event + .tags + .iter() + .filter(|t| t.as_slice().first().map(|s| s.as_str()) == Some("workflow-definition")) + .map(|t| t.as_slice()) + .collect(); + assert_eq!(definition_targets.len(), 1); + assert_eq!( + definition_targets[0], + [ + "workflow-definition", + definition.id.to_hex().as_str(), + "notify" + ] + ); + assert_eq!( + stored.event.content, "heads up @Robby — please take a look", + "ordinary workflow output must remain visible in the channel" + ); + let causes: Vec<&[String]> = stored + .event + .tags + .iter() + .filter(|tag| tag.as_slice().first().map(String::as_str) == Some("workflow-cause")) + .map(|tag| tag.as_slice()) + .collect(); + assert_eq!(causes.len(), 1); + assert_eq!(causes[0], ["workflow-cause", "webhook", ""]); + assert_eq!( + stored + .event + .tags + .iter() + .filter(|tag| tag.as_slice().first().map(String::as_str) == Some("workflow-run")) + .map(|tag| tag.as_slice()) + .collect::>(), + vec![["workflow-run", run_id.to_string().as_str()]], + "visible message must bind the immutable run consumed by ACP" + ); + assert_eq!( + stored + .event + .tags + .iter() + .filter(|tag| tag.as_slice().first().map(String::as_str) == Some("workflow-step")) + .map(|tag| tag.as_slice()) + .collect::>(), + vec![["workflow-step", "notify"]], + "visible message must bind the immutable step consumed by ACP" + ); + assert_eq!( + stored.event.pubkey, + state.relay_keypair.public_key(), + "workflow output must be signed by the relay identity" + ); + + assert!( + !stored.event.content.contains("relay-secret"), + "private webhook cargo must not enter the visible message" + ); + let later_trace = serde_json::json!([{ + "step_id": "call", + "output": {"body": "mutable-later-output"} + }]); + let later_trigger = serde_json::json!({ + "channel_id": channel.id.to_string(), + "definition_event_id": definition.id.to_hex(), + "webhook_fields": {"private_token": "mutable-later-secret"} + }); + sqlx::query("UPDATE workflow_runs SET trigger_context=$1 WHERE community_id=$2 AND id=$3") + .bind(&later_trigger) + .bind(community.as_uuid()) + .bind(run_id) + .execute(&pool) + .await + .expect("advance live run trigger context after delivery snapshot"); + state + .db + .update_workflow_run( + community, + run_id, + buzz_db::workflow::RunStatus::Running, + 2, + &later_trace, + None, + ) + .await + .expect("advance live run after delivery snapshot"); + + let delivery = state + .db + .claim_workflow_agent_delivery(community, &author.public_key().to_bytes(), None, None, 120) + .await + .expect("claim durable workflow delivery") + .expect("delivery exists for signed target"); + assert_eq!(delivery.run_id, run_id); + assert_eq!(delivery.step_id, "notify"); + assert_eq!(delivery.message_event_id, id_bytes); + assert_eq!( + delivery + .trigger_context + .as_ref() + .and_then(|value| value.get("webhook_fields")) + .and_then(|value| value.get("private_token")) + .and_then(serde_json::Value::as_str), + Some("relay-secret"), + "private webhook cargo must remain available only through the durable claim" + ); + assert_eq!(delivery.workflow_id, workflow_id); + assert_eq!(delivery.channel_id, channel.id); + assert_eq!(delivery.target_pubkey, author.public_key().to_bytes()); + assert_eq!(delivery.definition_event_id, definition.id.as_bytes()); + assert_eq!( + delivery.execution_trace, execution_trace, + "claim must return the immutable delivery snapshot, not the advanced run trace" + ); + assert_ne!(delivery.execution_trace, later_trace); + assert_eq!( + delivery.trigger_context.as_ref(), + Some(&trigger_json), + "claim must return the immutable delivery trigger snapshot" + ); + assert_ne!(delivery.trigger_context.as_ref(), Some(&later_trigger)); + + // Cross a Nostr timestamp second so signing again would necessarily create + // a distinct visible event. Durable identity must win before signing. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + let replay_event_id = sink + .send_message( + community, + workflow_id, + "notify", + &channel.id.to_string(), + "heads up @Robby — please take a look", + &author_hex, + &DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 2, + cause: WorkflowCause::Webhook, + }, + None, + ) + .await + .expect("replay same step"); + assert_eq!( + replay_event_id, event_id_hex, + "true replay across a timestamp second returns the canonical event" + ); + let visible_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM events WHERE community_id=$1 AND kind=9 AND id IN (SELECT message_event_id FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND step_id=$3)", + ) + .bind(community.as_uuid()) + .bind(run_id) + .bind("notify") + .fetch_one(&pool) + .await + .expect("count canonical visible events"); + assert_eq!( + visible_count, 1, + "delayed replay must not publish a duplicate" + ); + + let concurrent_step = "concurrent"; + let concurrent_doorbell = DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }; + let (left, right) = tokio::join!( + sink.send_message( + community, + workflow_id, + concurrent_step, + &channel.id.to_string(), + "parallel", + &author_hex, + &concurrent_doorbell, + None, + ), + sink.send_message( + community, + workflow_id, + concurrent_step, + &channel.id.to_string(), + "parallel", + &author_hex, + &concurrent_doorbell, + None, + ) + ); + assert_eq!( + left.expect("left concurrent send"), + right.expect("right concurrent send"), + "concurrent retries serialize on durable identity" + ); + let concurrent_visible_count: i64 = sqlx::query_scalar( + r#" + SELECT count(*) FROM events + WHERE community_id=$1 AND kind=9 + AND tags @> $2::jsonb + AND tags @> $3::jsonb + "#, + ) + .bind(community.as_uuid()) + .bind(serde_json::json!([["workflow-run", run_id.to_string()]])) + .bind(serde_json::json!([["workflow-step", concurrent_step]])) + .fetch_one(&pool) + .await + .expect("count concurrent visible events"); + assert_eq!(concurrent_visible_count, 1); + sqlx::query( + "UPDATE workflow_agent_deliveries SET status='delivered' WHERE community_id=$1 AND run_id=$2 AND step_id=$3", + ) + .bind(community.as_uuid()) + .bind(run_id) + .bind(concurrent_step) + .execute(&pool) + .await + .expect("complete concurrent regression delivery"); + + let follow_up_event_id = sink + .send_message( + community, + workflow_id, + "follow_up", + &channel.id.to_string(), + "previous=durable-prior-output", + &author_hex, + &DoorbellContext { + definition_event_id: definition.id.to_hex(), + run_id, + attempt: 1, + cause: WorkflowCause::Webhook, + }, + None, + ) + .await + .expect("send distinct step"); + assert_ne!( + follow_up_event_id, event_id_hex, + "distinct steps in one run remain distinct" + ); + let delivery_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM workflow_agent_deliveries WHERE community_id=$1 AND run_id=$2 AND target_pubkey=$3", + ) + .bind(community.as_uuid()) + .bind(run_id) + .bind(author.public_key().to_bytes()) + .fetch_one(&pool) + .await + .expect("count deliveries"); + assert_eq!( + delivery_count, 5, + "replay collapses while five distinct step identities survive" + ); + + let claim_token = delivery.claim_token.expect("claim has fencing token"); + let original_expiry = delivery.claim_expires_at.expect("claim has expiry"); + sqlx::query( + "UPDATE workflow_agent_deliveries SET claim_expires_at=NOW()+INTERVAL '1 second' WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .execute(&pool) + .await + .expect("move active claim to current lease boundary"); + let renewed_expiry = state + .db + .renew_workflow_agent_delivery( + community, + delivery.id, + &author.public_key().to_bytes(), + claim_token, + 7_600, + ) + .await + .expect("renew active claim") + .expect("current owner and token renew"); + assert!(renewed_expiry > original_expiry); + assert!(state + .db + .claim_workflow_agent_delivery( + community, + &author.public_key().to_bytes(), + Some(delivery.id), + None, + 120, + ) + .await + .expect("competing claim while renewed") + .is_none()); + assert!(state + .db + .renew_workflow_agent_delivery( + community, + delivery.id, + &author.public_key().to_bytes(), + Uuid::new_v4(), + 7_600, + ) + .await + .expect("stale renewal result") + .is_none()); + + sqlx::query( + "UPDATE workflow_agent_deliveries SET status='pending', claim_token=NULL, claim_owner=NULL, claim_expires_at=NULL, expires_at=NOW()+INTERVAL '119 seconds' WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .execute(&pool) + .await + .expect("leave less lifetime than requested lease"); + assert!( + state + .db + .claim_workflow_agent_delivery( + community, + &author.public_key().to_bytes(), + Some(delivery.id), + None, + 120, + ) + .await + .expect("short-lived claim admission") + .is_none(), + "claim admission never promises a lease beyond row expiry" + ); + sqlx::query( + "UPDATE workflow_agent_deliveries SET status='claimed', claim_token=$3, claim_owner=$4, claim_expires_at=NOW()+INTERVAL '7600 seconds', expires_at=NOW()+make_interval(secs => $5) WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .bind(claim_token) + .bind(author.public_key().to_bytes().as_slice()) + .bind(buzz_core::workflow_delivery::ROW_LIFETIME_SECONDS as f64) + .execute(&pool) + .await + .expect("restore active claim and durable row lifetime"); + + let row_expires_at: chrono::DateTime = sqlx::query_scalar( + "SELECT expires_at FROM workflow_agent_deliveries WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .fetch_one(&pool) + .await + .expect("read durable row lifetime"); + let required_lifetime = + chrono::Duration::seconds(buzz_core::workflow_delivery::ROW_LIFETIME_SECONDS); + assert!( + row_expires_at - chrono::Utc::now() >= required_lifetime - chrono::Duration::seconds(5), + "durable row must outlive the maximum admitted lease" + ); + + assert!(state + .db + .finish_workflow_agent_delivery( + community, + delivery.id, + &author.public_key().to_bytes(), + claim_token, + false, + true, + Some("agent_busy"), + Some("retry later"), + ) + .await + .expect("record retryable finish")); + sqlx::query( + "UPDATE workflow_agent_deliveries SET next_attempt_at=NOW() WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .execute(&pool) + .await + .expect("make retry due"); + let retry = state + .db + .claim_workflow_agent_delivery( + community, + &author.public_key().to_bytes(), + Some(delivery.id), + None, + 120, + ) + .await + .expect("retry claim") + .expect("retryable delivery becomes claimable"); + assert_eq!(retry.attempt, 2); + assert_ne!(retry.claim_token, Some(claim_token)); + + sqlx::query( + "UPDATE workflow_agent_deliveries SET claim_expires_at=NOW()-INTERVAL '1 second' WHERE community_id=$1 AND id=$2", + ) + .bind(community.as_uuid()) + .bind(delivery.id) + .execute(&pool) + .await + .expect("expire lease"); + assert!( + state + .db + .claim_workflow_agent_delivery( + community, + &author.public_key().to_bytes(), + Some(delivery.id), + None, + 120, + ) + .await + .expect("claim expired lease") + .is_none(), + "an expired exclusive claim is never reassigned" + ); + let terminal = state + .db + .reap_workflow_agent_deliveries() + .await + .expect("reap expired exclusive lease"); + assert!(terminal + .iter() + .any(|row| { row.id == delivery.id && row.status == "failed" && row.attempt == 2 })); + assert!(state + .db + .claim_workflow_agent_delivery( + community, + &author.public_key().to_bytes(), + Some(delivery.id), + None, + 120, + ) + .await + .expect("terminal claim result") + .is_none()); + + use axum::{ + body::{to_bytes, Body}, + http::{header, Request, StatusCode}, + }; + use tower::ServiceExt; + + let claim_body = serde_json::json!({"delivery_id": null}).to_string(); + let claim_response = crate::router::build_router(state.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri("/workflows/agent-deliveries/claim") + .header(header::HOST, &host) + .header("x-pubkey", &author_hex) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(claim_body)) + .expect("claim request"), + ) + .await + .expect("claim response"); + assert_eq!(claim_response.status(), StatusCode::OK); + let claim_json: serde_json::Value = serde_json::from_slice( + &to_bytes(claim_response.into_body(), 1024 * 1024) + .await + .expect("claim response body"), + ) + .expect("claim response JSON"); + let follow_up = &claim_json["delivery"]; + assert_eq!(follow_up["step_id"], "follow_up"); + assert_eq!(follow_up["workflow_id"], workflow_id.to_string()); + assert_eq!(follow_up["run_id"], run_id.to_string()); + let follow_up_id = follow_up["id"].as_str().unwrap().parse::().unwrap(); + let follow_up_token = follow_up["claim_token"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + + let finish_path = format!("/workflows/agent-deliveries/{follow_up_id}/finish"); + let finish_body = serde_json::json!({ + "claim_token": follow_up_token, + "delivered": true, + "retryable": false + }) + .to_string(); + let finish_response = crate::router::build_router(state.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri(&finish_path) + .header(header::HOST, &host) + .header("x-pubkey", &author_hex) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(finish_body.clone())) + .expect("finish request"), + ) + .await + .expect("finish response"); + assert_eq!(finish_response.status(), StatusCode::OK); + let finish_json: serde_json::Value = serde_json::from_slice( + &to_bytes(finish_response.into_body(), 1024 * 1024) + .await + .expect("finish response body"), + ) + .expect("finish response JSON"); + assert_eq!(finish_json["completed"], true); + + let stale_response = crate::router::build_router(state.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri(&finish_path) + .header(header::HOST, &host) + .header("x-pubkey", &author_hex) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(finish_body)) + .expect("stale finish request"), + ) + .await + .expect("replayed finish response"); + assert_eq!(stale_response.status(), StatusCode::OK); +} diff --git a/crates/buzz-test-client/tests/e2e_workflow_agent_owner.rs b/crates/buzz-test-client/tests/e2e_workflow_agent_owner.rs new file mode 100644 index 00000000000..41e0d4d4aab --- /dev/null +++ b/crates/buzz-test-client/tests/e2e_workflow_agent_owner.rs @@ -0,0 +1,161 @@ +//! End-to-end authorization coverage for a human operating an agent-owned workflow. +//! +//! Run against a local relay with: +//! `cargo test -p buzz-test-client --test e2e_workflow_agent_owner -- --ignored` + +use buzz_sdk::nip_oa; +use buzz_test_client::BuzzTestClient; +use nostr::{EventBuilder, Keys, Kind, Tag}; + +fn relay_url() -> String { + std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string()) +} + +fn workflow_yaml() -> String { + "name: Agent-owned workflow\n\ + trigger:\n\ + \x20 on: webhook\n\ + steps:\n\ + \x20 - id: notify\n\ + \x20 action: send_message\n\ + \x20 text: owner-triggered\n" + .to_string() +} + +async fn connect_agent_with_owner(agent: &Keys, owner: &Keys) -> BuzzTestClient { + let tag_json = + nip_oa::compute_auth_tag(owner, &agent.public_key(), "").expect("compute NIP-OA auth tag"); + let auth_tag = nip_oa::parse_auth_tag(&tag_json).expect("parse NIP-OA auth tag"); + let mut client = BuzzTestClient::connect_unauthenticated(&relay_url()) + .await + .expect("connect agent"); + client + .authenticate_with_nip_oa(agent, &auth_tag) + .await + .expect("authenticate agent with NIP-OA"); + client +} + +#[tokio::test] +#[ignore] +async fn agent_owner_can_trigger_but_unrelated_user_cannot() { + let owner = Keys::generate(); + let agent = Keys::generate(); + let unrelated = Keys::generate(); + let channel_id = uuid::Uuid::new_v4(); + let workflow_id = uuid::Uuid::new_v4(); + + // Authenticating the agent materializes the immutable community-scoped + // agent→owner relationship used by workflow authorization. + let mut agent_client = connect_agent_with_owner(&agent, &owner).await; + + let create_channel = EventBuilder::new(Kind::Custom(9007), "") + .tags(vec![ + Tag::parse(["h", &channel_id.to_string()]).unwrap(), + Tag::parse(["name", "workflow-agent-owner-e2e"]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", "open"]).unwrap(), + ]) + .sign_with_keys(&agent) + .unwrap(); + let ok = agent_client + .send_event(create_channel) + .await + .expect("create channel"); + assert!( + ok.accepted, + "agent channel creation rejected: {}", + ok.message + ); + + let define = buzz_sdk::build_workflow_def(channel_id, workflow_id, &workflow_yaml()) + .expect("build workflow definition") + .sign_with_keys(&agent) + .expect("sign workflow definition"); + let ok = agent_client + .send_event(define) + .await + .expect("define workflow"); + assert!( + ok.accepted, + "agent workflow definition rejected: {}", + ok.message + ); + + // SEC-006 boundary: channel membership alone must not grant workflow + // authority. Make the adversarial principal a real member before its + // trigger attempt below. + let add_unrelated = EventBuilder::new(Kind::Custom(9000), "") + .tags(vec![ + Tag::parse(["h", &channel_id.to_string()]).unwrap(), + Tag::parse(["p", &unrelated.public_key().to_string()]).unwrap(), + ]) + .sign_with_keys(&agent) + .expect("sign add-member event"); + let ok = agent_client + .send_event(add_unrelated) + .await + .expect("add unrelated user as channel member"); + assert!( + ok.accepted, + "adding unrelated channel member rejected: {}", + ok.message + ); + + let agent_trigger = buzz_sdk::build_workflow_trigger(workflow_id) + .expect("build agent trigger") + .sign_with_keys(&agent) + .expect("sign agent trigger"); + let ok = agent_client + .send_event(agent_trigger) + .await + .expect("send agent trigger"); + assert!( + ok.accepted, + "agent trigger of its own workflow rejected: {}", + ok.message + ); + + let mut unrelated_client = BuzzTestClient::connect(&relay_url(), &unrelated) + .await + .expect("connect unrelated user"); + let unrelated_trigger = buzz_sdk::build_workflow_trigger(workflow_id) + .expect("build unrelated trigger") + .sign_with_keys(&unrelated) + .expect("sign unrelated trigger"); + let ok = unrelated_client + .send_event(unrelated_trigger) + .await + .expect("send unrelated trigger"); + assert!( + !ok.accepted, + "unrelated user triggered an agent-owned workflow" + ); + assert!( + ok.message + .contains("not authorized to trigger this workflow"), + "unexpected rejection: {}", + ok.message + ); + + let mut owner_client = BuzzTestClient::connect(&relay_url(), &owner) + .await + .expect("connect owner"); + let owner_trigger = buzz_sdk::build_workflow_trigger(workflow_id) + .expect("build owner trigger") + .sign_with_keys(&owner) + .expect("sign owner trigger"); + let ok = owner_client + .send_event(owner_trigger) + .await + .expect("send owner trigger"); + assert!( + ok.accepted, + "owner trigger of agent-owned workflow rejected: {}", + ok.message + ); + + agent_client.disconnect().await.ok(); + unrelated_client.disconnect().await.ok(); + owner_client.disconnect().await.ok(); +} diff --git a/crates/buzz-workflow/src/action_sink.rs b/crates/buzz-workflow/src/action_sink.rs index 079c27a913d..2cd78774670 100644 --- a/crates/buzz-workflow/src/action_sink.rs +++ b/crates/buzz-workflow/src/action_sink.rs @@ -7,6 +7,22 @@ use std::future::Future; use std::pin::Pin; use buzz_core::tenant::CommunityId; +use uuid::Uuid; + +use crate::executor::WorkflowCause; + +/// Provenance attached to a workflow-generated agent doorbell. +#[derive(Debug, Clone)] +pub struct DoorbellContext { + /// Exact controller-signed kind:30620 definition revision. + pub definition_event_id: String, + /// Durable execution whose state authorizes this delivery. + pub run_id: Uuid, + /// Execution attempt for this step. + pub attempt: u32, + /// Semantic cause of this run. + pub cause: WorkflowCause, +} /// Errors from action sink operations. #[derive(Debug, thiserror::Error)] @@ -53,21 +69,27 @@ pub trait ActionSink: Send + Sync { /// under *this* community, never the deployment/default tenant — the run /// carries its owning community so a workflow in community B posts into B /// even though the side effect has no inbound connection to bind. + /// - `workflow_id`: UUID of the owner-signed kind:30620 definition + /// - `step_id`: ID of the `send_message` step being executed /// - `channel_id`: UUID string of the target channel /// - `text`: message body (must not be empty/whitespace-only) /// - `author_pubkey`: hex-encoded pubkey of the workflow owner (used for - /// the `p` attribution tag; the relay keypair signs the event) - /// - `reply_to`: when `Some(event_id_hex)`, the message is posted as a - /// threaded reply to that event (NIP-10 root/reply tags + real thread - /// metadata); when `None`, it is a top-level channel message. + /// the dedicated `workflow-owner` authority tag and a `p` attribution + /// tag; the relay keypair signs the event) + /// - `reply_to`: when present, reply to this triggering message while + /// preserving NIP-10 ancestry and durable thread metadata /// /// Returns the event ID hex string on success. + #[allow(clippy::too_many_arguments)] fn send_message( &self, community_id: CommunityId, + workflow_id: Uuid, + step_id: &str, channel_id: &str, text: &str, author_pubkey: &str, + doorbell: &DoorbellContext, reply_to: Option<&str>, ) -> Pin> + Send + '_>>; } diff --git a/crates/buzz-workflow/src/executor.rs b/crates/buzz-workflow/src/executor.rs index 5c712dcff7c..05e21064459 100644 --- a/crates/buzz-workflow/src/executor.rs +++ b/crates/buzz-workflow/src/executor.rs @@ -18,6 +18,7 @@ use serde_json::Value as JsonValue; use tracing::{debug, info, warn}; use uuid::Uuid; +use crate::action_sink::DoorbellContext; use crate::error::WorkflowError; use crate::schema::{ActionDef, Step, WorkflowDef}; use crate::WorkflowEngine; @@ -43,6 +44,26 @@ pub struct TriggerContext { pub is_reply: bool, /// Arbitrary webhook body fields (webhook trigger). pub webhook_fields: HashMap, + /// Exact owner-signed kind:30620 definition revision executed by this run. + #[serde(default)] + pub definition_event_id: String, + /// Semantic cause carried to workflow-generated agent doorbells. + #[serde(default)] + pub cause: Option, +} + +/// Provenance for the event that caused a workflow run. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum WorkflowCause { + /// Signed message, reaction, or diff event. + Event(String), + /// Deterministic UTC schedule slot. + Schedule(String), + /// Owner-signed kind:46020 manual-run command. + Command(String), + /// Unsigned external webhook payload. + Webhook, } impl TriggerContext { @@ -622,13 +643,31 @@ pub async fn dispatch_action( "SendMessage → {channel_id}: {text}" ); + let doorbell = DoorbellContext { + definition_event_id: trigger_ctx.definition_event_id.clone(), + run_id, + attempt: 1, + cause: trigger_ctx.cause.clone().ok_or_else(|| { + WorkflowError::InvalidDefinition( + "SendMessage: workflow cause provenance is unavailable".into(), + ) + })?, + }; + if doorbell.definition_event_id.is_empty() { + return Err(WorkflowError::InvalidDefinition( + "SendMessage: workflow definition provenance is unavailable".into(), + )); + } let event_id = engine .action_sink()? .send_message( community_id, + workflow.id, + step_id, &channel_id, text, &owner_pubkey_hex, + &doorbell, reply_to, ) .await @@ -1263,6 +1302,28 @@ async fn execute_steps( "output": output, })); step_outputs.insert(step.id.clone(), output); + // Checkpoint every completed step. A later agent wake can then + // reconstruct prior outputs from durable relay-owned run state. + engine + .db + .update_workflow_run( + community_id, + run_id, + buzz_db::workflow::RunStatus::Running, + (i + 1) as i32, + &serde_json::Value::Array(trace.clone()), + None, + ) + .await + .map_err(|error| { + ( + WorkflowError::from(error), + crate::error::PartialProgress { + step_index: i, + trace: trace.clone(), + }, + ) + })?; } StepResult::Suspended { approval_token } => { info!( @@ -1312,6 +1373,8 @@ mod tests { message_id: "event-id-hex".to_owned(), is_reply: false, webhook_fields: HashMap::new(), + definition_event_id: String::new(), + cause: None, } } diff --git a/crates/buzz-workflow/src/lib.rs b/crates/buzz-workflow/src/lib.rs index bceb6d8bd8d..db36db033a8 100644 --- a/crates/buzz-workflow/src/lib.rs +++ b/crates/buzz-workflow/src/lib.rs @@ -359,14 +359,6 @@ impl WorkflowEngine { let trigger_ctx = build_trigger_context(event); - let trigger_ctx_json: serde_json::Value = match serde_json::to_value(&trigger_ctx) { - Ok(v) => v, - Err(e) => { - tracing::error!("Failed to serialize trigger context: {e}"); - return Ok(()); - } - }; - for workflow in workflows.iter() { let def: WorkflowDef = match serde_json::from_value(workflow.definition.clone()) { Ok(d) => d, @@ -401,6 +393,22 @@ impl WorkflowEngine { continue; } + let Some(definition_event_id) = workflow.definition_event_id.as_deref() else { + tracing::warn!(workflow_id = %workflow.id, "Skipping workflow — owner-signed definition revision is unavailable"); + continue; + }; + let definition_event_id = hex::encode(definition_event_id); + let mut workflow_trigger_ctx = trigger_ctx.clone(); + workflow_trigger_ctx.definition_event_id = definition_event_id.clone(); + workflow_trigger_ctx.cause = + Some(executor::WorkflowCause::Event(event.event.id.to_hex())); + let workflow_trigger_ctx_json = match serde_json::to_value(&workflow_trigger_ctx) { + Ok(value) => value, + Err(error) => { + tracing::warn!(workflow_id = %workflow.id, "Failed to serialize workflow provenance: {error}"); + continue; + } + }; let trigger_event_id_bytes = event.event.id.as_bytes().to_vec(); let run_id = match self .db @@ -408,7 +416,7 @@ impl WorkflowEngine { community_id, workflow.id, Some(&trigger_event_id_bytes), - Some(&trigger_ctx_json), + Some(&workflow_trigger_ctx_json), ) .await { @@ -427,7 +435,7 @@ impl WorkflowEngine { let engine = Arc::clone(self); let def_clone = def.clone(); - let ctx_clone = trigger_ctx.clone(); + let ctx_clone = workflow_trigger_ctx; tokio::spawn(async move { let result = @@ -648,9 +656,17 @@ impl WorkflowEngine { // Fix 5: handle serialization errors explicitly rather than silently // dropping the trigger context with .ok(). + let Some(definition_event_id) = workflow.definition_event_id.as_deref() else { + tracing::warn!(workflow_id = %workflow.id, "Cron tick: owner-signed definition revision is unavailable"); + continue; + }; let trigger_ctx = executor::TriggerContext { channel_id: channel_id.to_string(), - timestamp: now.timestamp().to_string(), + timestamp: scheduled_for.timestamp().to_string(), + definition_event_id: hex::encode(definition_event_id), + cause: Some(executor::WorkflowCause::Schedule( + scheduled_for.to_rfc3339(), + )), ..Default::default() }; let trigger_ctx_json = match serde_json::to_value(&trigger_ctx) { @@ -875,6 +891,42 @@ fn interval_prefilter_should_fire( false } +/// Validate a relay-provided schedule cause against the signed definition. +/// Freshness/skew is intentionally a separate policy: this checks only that +/// the slot is an exact cron occurrence or interval boundary. +pub fn schedule_cause_matches(def: &WorkflowDef, slot: DateTime) -> bool { + match &def.trigger { + schema::TriggerDef::Schedule { + cron: Some(expr), + interval: None, + } => { + let Ok(schedule) = schema::normalize_cron(expr).parse::() else { + return false; + }; + let previous = slot - chrono::Duration::seconds(1); + schedule.after(&previous).next() == Some(slot) + } + schema::TriggerDef::Schedule { + cron: None, + interval: Some(duration), + } => executor::parse_duration_secs(duration) + .ok() + .is_some_and(|seconds| seconds > 0 && slot.timestamp().rem_euclid(seconds as i64) == 0), + _ => false, + } +} + +/// Validate that a signed source event semantically satisfies a workflow trigger. +/// Used by ACP doorbell verification after independently refetching the source. +pub async fn trigger_matches_signed_event( + def: &WorkflowDef, + trigger_ctx: &executor::TriggerContext, + kind_u32: u32, +) -> bool { + trigger_matches_event(&def.trigger, kind_u32) + && should_fire_workflow(def, trigger_ctx, Uuid::nil()).await +} + /// Check emoji and filter-expression conditions that determine whether a /// matched workflow should actually fire. Extracted from `on_event` to keep /// the per-workflow loop body small. @@ -999,6 +1051,8 @@ pub fn build_trigger_context(event: &buzz_core::StoredEvent) -> executor::Trigge message_id, is_reply: event_is_reply(&event.event), webhook_fields: HashMap::new(), + definition_event_id: String::new(), + cause: None, } } @@ -1050,6 +1104,39 @@ fn trigger_matches_event(trigger: &TriggerDef, kind_u32: u32) -> bool { mod tests { use super::*; + #[test] + fn schedule_cause_requires_exact_cron_or_interval_slot() { + let cron: WorkflowDef = serde_json::from_value(serde_json::json!({ + "name": "cron", + "trigger": { "on": "schedule", "cron": "0 * * * *" }, + "steps": [{ "id": "wake", "action": "send_message", "text": "wake" }], + "enabled": true + })) + .unwrap(); + let on_hour = DateTime::parse_from_rfc3339("2026-08-14T15:00:00Z") + .unwrap() + .with_timezone(&Utc); + assert!(schedule_cause_matches(&cron, on_hour)); + assert!(!schedule_cause_matches( + &cron, + on_hour + chrono::Duration::seconds(1) + )); + + let interval: WorkflowDef = serde_json::from_value(serde_json::json!({ + "name": "interval", + "trigger": { "on": "schedule", "interval": "5m" }, + "steps": [{ "id": "wake", "action": "send_message", "text": "wake" }], + "enabled": true + })) + .unwrap(); + let boundary = DateTime::from_timestamp(1_800_000_000, 0).unwrap(); + assert!(schedule_cause_matches(&interval, boundary)); + assert!(!schedule_cause_matches( + &interval, + boundary + chrono::Duration::seconds(1) + )); + } + #[test] fn cron_fire_instant_matches_within_window() { // "every minute" cron — should always fire within a 60s window. diff --git a/crates/buzz-workflow/src/schema.rs b/crates/buzz-workflow/src/schema.rs index 0e8dfdb52ef..b6206388c07 100644 --- a/crates/buzz-workflow/src/schema.rs +++ b/crates/buzz-workflow/src/schema.rs @@ -211,6 +211,18 @@ impl WorkflowDef { step.id ))); } + if let ActionDef::SendMessage { + channel: Some(channel), + .. + } = &step.action + { + if !channel.trim().is_empty() { + return Err(WorkflowError::InvalidDefinition( + "send_message channel overrides are not supported; workflows are bound to their definition channel" + .into(), + )); + } + } } // `reply_in_thread` requires a triggering message to reply to. Schedule @@ -389,7 +401,7 @@ mod tests { "name: All Actions\n", "trigger:\n on: webhook\n", "steps:\n", - " - id: msg\n action: send_message\n text: Hello\n channel: general\n", + " - id: msg\n action: send_message\n text: Hello\n", " - id: dm\n action: send_dm\n to: '{{trigger.author}}'\n text: You triggered this\n", " - id: topic\n action: set_channel_topic\n topic: Status active\n", " - id: react\n action: add_reaction\n emoji: white_check_mark\n", @@ -876,6 +888,20 @@ mod tests { assert_eq!(def.steps.len(), 3); } + #[test] + fn validate_rejects_send_message_channel_override() { + let yaml = concat!( + "name: Cross Channel\ntrigger:\n on: message_posted\n", + "steps:\n - id: notify\n action: send_message\n", + " text: hi\n channel: 11111111-1111-1111-1111-111111111111\n", + ); + let error = parse_yaml(yaml).unwrap_err(); + assert!( + matches!(&error, WorkflowError::InvalidDefinition(message) if message.contains("channel overrides are not supported")), + "unexpected error: {error}" + ); + } + #[test] fn step_id_validation_rejects_dashes() { // Step ID with dash would cause evalexpr to interpret as subtraction: diff --git a/docs/workflow-agent-delivery.md b/docs/workflow-agent-delivery.md new file mode 100644 index 00000000000..cf6b8e11c86 --- /dev/null +++ b/docs/workflow-agent-delivery.md @@ -0,0 +1,10 @@ +# Managed-agent workflow delivery + +A workflow `send_message` step has two separate outputs when it addresses a managed agent: + +1. The relay signs and persists an ordinary kind `9` channel message. Its content is the rendered, human-visible step text. The event carries the channel, exact kind `30620` definition revision and step, workflow owner, semantic cause, and signed-template-derived `p` recipients. Private webhook fields and prior-step state are never copied into this event. +2. The relay creates a durable `workflow_agent_deliveries` row for each signed routing target, bound by foreign key to the exact partitioned kind `9` event. It then publishes an ephemeral kind `24620` wake hint containing only identifiers. Kind `24620` is `p`-gated; it is an accelerator rather than the source of truth. + +The managed agent claims a delivery through the authenticated relay endpoint. Claiming is scoped to the host-derived community and authenticated agent pubkey, returns a fenced lease token, and includes the private trigger and execution snapshot. Before dispatch, ACP independently fetches and verifies the exact definition and visible message, immutable NIP-OA ownership, channel and workflow/run/step bindings, semantic cause, and locally rendered text. A stale or unverifiable delivery fails closed. + +ACP also polls for pending deliveries, so an offline agent does not depend on receiving the ephemeral wake. Completion uses the lease token. Only an authenticated, token-fenced retryable finish returns a live claim to pending with bounded backoff. Once claimed, ACP is the sole retry authority: lease expiry is terminal and never redelivers, choosing at-most-once execution over potentially duplicating successful agent side effects after an uncertain finish or runtime exit. Pending rows that expire before claim also become terminal. The uniqueness key `(community_id, run_id, step_id, target_pubkey)` collapses a true replay while preserving distinct steps in the same run. diff --git a/migrations/0033_workflow_definition_event_id.sql b/migrations/0033_workflow_definition_event_id.sql new file mode 100644 index 00000000000..4405fc21f49 --- /dev/null +++ b/migrations/0033_workflow_definition_event_id.sql @@ -0,0 +1,4 @@ +-- Bind each materialized workflow row to the exact owner-signed kind:30620 +-- revision that produced it. Existing rows remain nullable until re-saved; +-- workflow doorbells fail closed when the revision is unavailable. +ALTER TABLE workflows ADD COLUMN definition_event_id BYTEA; diff --git a/migrations/0034_workflow_agent_deliveries.sql b/migrations/0034_workflow_agent_deliveries.sql new file mode 100644 index 00000000000..6de4ea68334 --- /dev/null +++ b/migrations/0034_workflow_agent_deliveries.sql @@ -0,0 +1,45 @@ +-- Durable handoff from a workflow send_message step to a managed agent. +CREATE TYPE workflow_agent_delivery_status AS ENUM ('pending', 'claimed', 'delivered', 'failed', 'expired'); + +CREATE TABLE workflow_agent_deliveries ( + community_id UUID NOT NULL REFERENCES communities(id), + id UUID NOT NULL, + workflow_id UUID NOT NULL, + run_id UUID NOT NULL, + step_id VARCHAR(64) NOT NULL, + definition_event_id BYTEA NOT NULL, + message_event_id BYTEA NOT NULL, + message_event_created_at TIMESTAMPTZ NOT NULL, + channel_id UUID NOT NULL, + target_pubkey BYTEA NOT NULL, + status workflow_agent_delivery_status NOT NULL DEFAULT 'pending', + attempt INT NOT NULL DEFAULT 0 CHECK (attempt BETWEEN 0 AND 3), + claim_token UUID, + claim_owner BYTEA, + claim_expires_at TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + delivered_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + failure_code TEXT, + failure_message TEXT, + -- Immutable private execution snapshot used to verify rendering. + execution_trace JSONB NOT NULL, + trigger_context JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (community_id, id), + UNIQUE (community_id, run_id, step_id, target_pubkey), + FOREIGN KEY (community_id, workflow_id) REFERENCES workflows (community_id, id) ON DELETE CASCADE, + FOREIGN KEY (community_id, run_id) REFERENCES workflow_runs (community_id, id) ON DELETE CASCADE, + FOREIGN KEY (community_id, message_event_created_at, message_event_id) + REFERENCES events (community_id, created_at, id) ON DELETE CASCADE +); + +CREATE INDEX idx_workflow_agent_deliveries_pending + ON workflow_agent_deliveries (community_id, target_pubkey, next_attempt_at, created_at) + WHERE status IN ('pending', 'claimed'); +CREATE INDEX idx_workflow_agent_deliveries_run + ON workflow_agent_deliveries (community_id, run_id, step_id); + +SELECT attach_community_write_fence('workflow_agent_deliveries'); diff --git a/schema/schema.sql b/schema/schema.sql index 6e14e6be1bf..e767cf20f20 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -31,6 +31,7 @@ CREATE TYPE member_role AS ENUM ('owner', 'admin', 'member', 'guest', 'bot'); CREATE TYPE workflow_status AS ENUM ('active', 'disabled', 'archived'); CREATE TYPE run_status AS ENUM ('pending', 'running', 'waiting_approval', 'completed', 'failed', 'cancelled'); CREATE TYPE approval_status AS ENUM ('pending', 'granted', 'denied', 'expired'); +CREATE TYPE workflow_agent_delivery_status AS ENUM ('pending', 'claimed', 'delivered', 'failed', 'expired'); CREATE TYPE delivery_method AS ENUM ('webhook', 'websocket'); CREATE TYPE subscription_status AS ENUM ('active', 'paused', 'deleted'); CREATE TYPE pause_reason AS ENUM ('user', 'system', 'rate_limit'); @@ -368,6 +369,9 @@ CREATE TABLE workflows ( channel_id UUID, definition JSONB NOT NULL, definition_hash BYTEA NOT NULL, + -- Exact owner-signed kind:30620 revision that materialized this row. + -- Nullable only for pre-0032 rows; workflow doorbells fail closed until re-saved. + definition_event_id BYTEA, status workflow_status NOT NULL DEFAULT 'active', enabled BOOLEAN NOT NULL DEFAULT TRUE, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), @@ -406,6 +410,48 @@ CREATE TABLE workflow_runs ( CREATE INDEX idx_workflow_runs_workflow ON workflow_runs (community_id, workflow_id); CREATE INDEX idx_workflow_runs_status ON workflow_runs (community_id, status); +-- ── Durable workflow-agent deliveries ──────────────────────────────────────── +CREATE TABLE workflow_agent_deliveries ( + community_id UUID NOT NULL REFERENCES communities(id), + id UUID NOT NULL, + workflow_id UUID NOT NULL, + run_id UUID NOT NULL, + step_id VARCHAR(64) NOT NULL, + definition_event_id BYTEA NOT NULL, + message_event_id BYTEA NOT NULL, + message_event_created_at TIMESTAMPTZ NOT NULL, + channel_id UUID NOT NULL, + target_pubkey BYTEA NOT NULL, + status workflow_agent_delivery_status NOT NULL DEFAULT 'pending', + attempt INT NOT NULL DEFAULT 0 CHECK (attempt BETWEEN 0 AND 3), + claim_token UUID, + claim_owner BYTEA, + claim_expires_at TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + delivered_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + failure_code TEXT, + failure_message TEXT, + execution_trace JSONB NOT NULL, + trigger_context JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (community_id, id), + UNIQUE (community_id, run_id, step_id, target_pubkey), + FOREIGN KEY (community_id, workflow_id) REFERENCES workflows (community_id, id) ON DELETE CASCADE, + FOREIGN KEY (community_id, run_id) REFERENCES workflow_runs (community_id, id) ON DELETE CASCADE, + FOREIGN KEY (community_id, message_event_created_at, message_event_id) + REFERENCES events (community_id, created_at, id) ON DELETE CASCADE +); + +CREATE INDEX idx_workflow_agent_deliveries_pending + ON workflow_agent_deliveries (community_id, target_pubkey, next_attempt_at, created_at) + WHERE status IN ('pending', 'claimed'); +CREATE INDEX idx_workflow_agent_deliveries_run + ON workflow_agent_deliveries (community_id, run_id, step_id); + + -- ── Workflow approvals ──────────────────────────────────────────────────────── -- token-hash lookup scoped: approval token grants cannot act on another -- community's same hash (conformance). @@ -1744,6 +1790,7 @@ SELECT attach_community_write_fence('scheduled_workflow_fires'); SELECT attach_community_write_fence('subscriptions'); SELECT attach_community_write_fence('thread_metadata'); SELECT attach_community_write_fence('users'); +SELECT attach_community_write_fence('workflow_agent_deliveries'); SELECT attach_community_write_fence('workflow_approvals'); SELECT attach_community_write_fence('workflow_runs'); SELECT attach_community_write_fence('workflows');