diff --git a/Cargo.toml b/Cargo.toml index 79c66361b..1b4e22322 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,13 @@ exclude = ["tests/e2e-ui"] resolver = "2" [workspace.package] -version = "0.1.0" +version = "1.0.0" edition = "2021" license = "MIT" repository = "https://github.com/patrickleet/distributed" [workspace.dependencies.distributed_macros] -version = "0.1.0" +version = "1.0.0" path = "distributed_macros" [package] diff --git a/distributed_cli/Cargo.toml b/distributed_cli/Cargo.toml index 72130a91c..d5d45ff86 100644 --- a/distributed_cli/Cargo.toml +++ b/distributed_cli/Cargo.toml @@ -15,6 +15,7 @@ name = "distributed" path = "src/main.rs" [dependencies] +distributed = { path = "..", default-features = false } async-graphql-parser = "7" async-graphql-value = "7" base64 = "0.22.1" diff --git a/distributed_cli/src/cli.rs b/distributed_cli/src/cli.rs index 6735a65f4..9bfe4f0c6 100644 --- a/distributed_cli/src/cli.rs +++ b/distributed_cli/src/cli.rs @@ -31,8 +31,10 @@ use crate::{ MetricsTarget, PostCreateAction, ServiceScaffoldSpec, ServiceTransport, StoreTarget, }; -const DISTRIBUTED_MANIFEST_SCHEMA_VERSION: u64 = 1; -const DISTRIBUTED_CLIENT_MANIFEST_VERSION: u64 = 2; +const DISTRIBUTED_MANIFEST_SCHEMA_VERSION: u64 = + distributed::APPLICATION_MANIFEST_SCHEMA_VERSION as u64; +const DISTRIBUTED_CLIENT_MANIFEST_VERSION: u64 = + distributed::graphql::DISTRIBUTED_CLIENT_MANIFEST_VERSION as u64; /// Top-level standalone CLI arguments for the `distributed` binary. #[derive(Args, Debug)] @@ -58,6 +60,8 @@ pub enum DistributedCommands { Schema(SchemaArgs), /// Extract the embedded Distributed agent skills into a project Skills(SkillsArgs), + /// Validate and render a resolved deployment from one manifest + plan + Deployment(DeploymentArgs), } /// Library adapter for embedding service-related commands under another CLI. @@ -133,6 +137,40 @@ pub enum ContractsOutput { Json, } +#[derive(Args, Debug)] +pub struct DeploymentArgs { + #[command(subcommand)] + pub command: DeploymentCommands, +} + +#[derive(Subcommand, Debug)] +pub enum DeploymentCommands { + /// Validate that one manifest + plan resolve + Validate(DeploymentValidateArgs), + /// Render kubernetes, knative, or hops-xr from the same pair + Render(DeploymentRenderArgs), +} + +#[derive(Args, Debug)] +pub struct DeploymentValidateArgs { + #[arg(long)] + pub manifest: PathBuf, + #[arg(long)] + pub plan: PathBuf, +} + +#[derive(Args, Debug)] +pub struct DeploymentRenderArgs { + #[arg(long)] + pub manifest: PathBuf, + #[arg(long)] + pub plan: PathBuf, + #[arg(long)] + pub target: String, + #[arg(long, default_value = "dist")] + pub out: PathBuf, +} + #[derive(Args, Debug)] pub struct SkillsArgs { #[command(subcommand)] @@ -516,6 +554,44 @@ impl From for GitopsPromoteTarget { } } +fn run_deployment(args: &DeploymentArgs) -> Result<(), Box> { + match &args.command { + DeploymentCommands::Validate(validate) => { + let resolved = load_resolved(&validate.manifest, &validate.plan)?; + println!( + "deployment validate: ok application={} plan={} digest={}", + resolved.application, resolved.plan, resolved.inventory_digest + ); + Ok(()) + } + DeploymentCommands::Render(render) => { + let resolved = load_resolved(&render.manifest, &render.plan)?; + let target = distributed::RenderTarget::parse(&render.target)?; + let files = distributed::render_resolved(&resolved, target)?; + fs::create_dir_all(&render.out)?; + for file in files { + let path = render.out.join(&file.path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&path, file.contents.as_bytes())?; + println!("{}", path.display()); + } + Ok(()) + } + } +} + +fn load_resolved( + manifest_path: &Path, + plan_path: &Path, +) -> Result> { + let manifest: distributed::ApplicationManifest = + serde_json::from_str(&fs::read_to_string(manifest_path)?)?; + let plan: distributed::DeploymentPlan = serde_json::from_str(&fs::read_to_string(plan_path)?)?; + Ok(distributed::resolve_deployment(&manifest, &plan)?) +} + /// Dispatch the standalone `distributed` binary command tree. pub fn run_distributed(args: &DistributedArgs) -> Result<(), Box> { match &args.command { @@ -529,6 +605,7 @@ pub fn run_distributed(args: &DistributedArgs) -> Result<(), Box> { SkillsCommands::Init(init) => run_skills_init(init), SkillsCommands::List => run_skills_list(), }, + DistributedCommands::Deployment(deployment) => run_deployment(deployment), } } diff --git a/distributed_cli/src/client_compiler/manifest/mod.rs b/distributed_cli/src/client_compiler/manifest/mod.rs index 92e747912..fc2e34d52 100644 --- a/distributed_cli/src/client_compiler/manifest/mod.rs +++ b/distributed_cli/src/client_compiler/manifest/mod.rs @@ -15,10 +15,23 @@ pub(crate) use projectors::*; pub(crate) use roots::*; pub(crate) use util::{validate_hash, validate_nonempty}; -const MANIFEST_VERSION: u64 = 2; -const PROTOCOL_VERSION: u64 = 1; -const PROTOCOL_FINGERPRINT: &str = - "sha256:00fb342f3acb4dc1c1716a43cc3001c748d5f6c500ff831690d820e9e43e2782"; +const MANIFEST_VERSION: u64 = + distributed::graphql::DISTRIBUTED_CLIENT_MANIFEST_VERSION as u64; +const PROTOCOL_VERSION: u64 = + distributed::graphql::DISTRIBUTED_CLIENT_PROTOCOL_VERSION as u64; + +pub(crate) fn protocol_fingerprint() -> Result { + distributed::graphql::protocol_fingerprint().map_err(|error| { + ClientCompileError::manifest( + "client.manifest.protocol_fingerprint", + error.to_string(), + ) + }) +} + +pub(crate) fn expected_manifest_version() -> u64 { + MANIFEST_VERSION +} pub(crate) const CLIENT_PROJECTION_PROGRAM_VERSION: u32 = 2; pub(crate) const CLIENT_PROJECTION_BINDING_VERSION: u32 = 1; diff --git a/distributed_cli/src/client_compiler/manifest/parse.rs b/distributed_cli/src/client_compiler/manifest/parse.rs index c9651b271..86f6cc746 100644 --- a/distributed_cli/src/client_compiler/manifest/parse.rs +++ b/distributed_cli/src/client_compiler/manifest/parse.rs @@ -82,11 +82,12 @@ impl ClientManifest { validate_hash(&wire.schema_fingerprint, "manifest.schema_fingerprint")?; validate_hash(&wire.protocol_fingerprint, "manifest.protocol_fingerprint")?; validate_execution_limits(&wire.execution)?; - if wire.protocol_fingerprint != PROTOCOL_FINGERPRINT { + let expected_protocol = protocol_fingerprint()?; + if wire.protocol_fingerprint != expected_protocol { return Err(ClientCompileError::manifest( "client.manifest.protocol_fingerprint", format!( - "client compiler protocol contract is `{PROTOCOL_FINGERPRINT}`, received `{}`; regenerate the manifest and use a matching distributed version", + "client compiler protocol contract is `{expected_protocol}`, received `{}`; regenerate the manifest and use a matching distributed version", wire.protocol_fingerprint ), )); diff --git a/distributed_cli/src/client_compiler/mod.rs b/distributed_cli/src/client_compiler/mod.rs index 741806183..a4b2ec089 100644 --- a/distributed_cli/src/client_compiler/mod.rs +++ b/distributed_cli/src/client_compiler/mod.rs @@ -21,6 +21,14 @@ use serde_json::Value as JsonValue; use graphql::{compile_document, CompiledOperation}; use manifest::ClientManifest; + +pub(crate) fn expected_protocol_fingerprint() -> Result { + manifest::protocol_fingerprint() +} + +pub(crate) fn expected_manifest_version() -> u64 { + manifest::expected_manifest_version() +} use render::render_project; /// Complete input to one deterministic client compilation. diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index 127f38378..1081b6b1d 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1616,3 +1616,15 @@ fn unavailable_merge_base_is_reported_not_verified() { assert!(result.human().contains("history evidence unavailable")); assert!(result.human().contains("merge_base_available=false")); } + +#[test] +fn cli_protocol_identity_is_the_library_function() { + let library = distributed::graphql::protocol_fingerprint().expect("library fingerprint"); + let compiler = crate::client_compiler::expected_protocol_fingerprint() + .expect("compiler uses the same library function"); + assert_eq!(library, compiler); + assert_eq!( + u64::from(distributed::graphql::DISTRIBUTED_CLIENT_MANIFEST_VERSION), + crate::client_compiler::expected_manifest_version() + ); +} diff --git a/distributed_cli/src/lib.rs b/distributed_cli/src/lib.rs index 8c25387d4..8730b90cb 100644 --- a/distributed_cli/src/lib.rs +++ b/distributed_cli/src/lib.rs @@ -16,7 +16,8 @@ mod skills; pub use atlas::{render_atlas_schema, AtlasDatabaseUrl, AtlasSchemaSpec}; pub use cli::{ run, run_distributed, AgentHarness, Bus, ClientArgs, ClientManifestArgs, ContractsAcceptArgs, - ContractsArgs, ContractsCheckArgs, ContractsCommands, ContractsOutput, DescribeArgs, + ContractsArgs, ContractsCheckArgs, ContractsCommands, ContractsOutput, DeploymentArgs, + DeploymentCommands, DeploymentRenderArgs, DeploymentValidateArgs, DescribeArgs, DistributedArgs, DistributedCommands, Framework, GitopsPromote, ManifestFormat, Metrics, ScaffoldArgs, SchemaArgs, SchemaDialect, SchemaFormat, ServiceArgs, ServiceCommands, SkillsArgs, SkillsCommands, SkillsInitArgs, Store, Transport, diff --git a/distributed_cli/tests/cli_deployment.rs b/distributed_cli/tests/cli_deployment.rs new file mode 100644 index 000000000..fd14abfdc --- /dev/null +++ b/distributed_cli/tests/cli_deployment.rs @@ -0,0 +1,129 @@ +//! Drive the shipped `distributed` binary for deployment validate/render. + +use distributed::application::{ + compile_deployment_plan, Application, CommandDefinition, CommandSpec, CommandTypeSpec, + ModelFieldSpec, ModelSpec, Module, ProcessIntent, ProcessPreset, ProjectionSpec, +}; +use distributed::graphql::CommandConsistency; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn sample_pair() -> (distributed::ApplicationManifest, distributed::DeploymentPlan) { + let model = ModelSpec::try_new( + "TodoView", + "todos", + [ModelFieldSpec { + name: "todo_id".into(), + scalar: "String".into(), + nullable: false, + }], + ["todo_id"], + ) + .unwrap(); + let command = CommandSpec::try_new( + "todo.create", + "todo_create", + CommandTypeSpec { + name: "In".into(), + fields: vec![], + }, + CommandTypeSpec { + name: "Out".into(), + fields: vec![], + }, + CommandConsistency::Eventual, + ) + .unwrap(); + let module = Module::new("todo") + .command_definitions([CommandDefinition::contract(command)]) + .models([model]) + .projections([ + ProjectionSpec::try_new("project_todos", ["todo.created"], ["TodoView"]).unwrap(), + ]) + .build() + .unwrap(); + let manifest = Application::new("todo-app") + .module(module) + .build() + .unwrap() + .manifest() + .clone(); + let plan = compile_deployment_plan( + "split", + &manifest, + [ + ProcessIntent::with_preset("writer", &manifest, ProcessPreset::Writer).unwrap(), + ProcessIntent::with_preset("projector", &manifest, ProcessPreset::Projector).unwrap(), + ], + ) + .unwrap(); + (manifest, plan) +} + +fn write_pair(dir: &Path) -> (PathBuf, PathBuf) { + let (manifest, plan) = sample_pair(); + let manifest_path = dir.join("manifest.json"); + let plan_path = dir.join("plan.json"); + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest).unwrap()).unwrap(); + fs::write(&plan_path, serde_json::to_vec_pretty(&plan).unwrap()).unwrap(); + (manifest_path, plan_path) +} + +fn distributed() -> Command { + Command::new(env!("CARGO_BIN_EXE_distributed")) +} + +#[test] +fn deployment_validate_and_render_use_one_pair() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("cli-deployment"); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let (manifest, plan) = write_pair(&root); + + let validate = distributed() + .args([ + "deployment", + "validate", + "--manifest", + manifest.to_str().unwrap(), + "--plan", + plan.to_str().unwrap(), + ]) + .output() + .expect("distributed binary"); + assert!( + validate.status.success(), + "{}", + String::from_utf8_lossy(&validate.stderr) + ); + + for target in ["kubernetes", "knative", "hops-xr"] { + let out = root.join(target); + let render = distributed() + .args([ + "deployment", + "render", + "--manifest", + manifest.to_str().unwrap(), + "--plan", + plan.to_str().unwrap(), + "--target", + target, + "--out", + out.to_str().unwrap(), + ]) + .output() + .expect("distributed binary"); + assert!( + render.status.success(), + "{target}: {}", + String::from_utf8_lossy(&render.stderr) + ); + assert!( + out.join("deploy").exists(), + "{target} wrote {}", + String::from_utf8_lossy(&render.stdout) + ); + } +} diff --git a/src/application/mod.rs b/src/application/mod.rs index c048e1098..c1177dfa0 100644 --- a/src/application/mod.rs +++ b/src/application/mod.rs @@ -15,6 +15,8 @@ mod module; mod mount; mod plan; mod registration; +mod render; +mod resolve; mod runtime_host; mod topology; @@ -48,6 +50,12 @@ pub use plan::{ DEPLOYMENT_PLAN_SCHEMA_VERSION, MAX_DEPLOYMENT_PLAN_BYTES, }; pub use registration::{Application, ApplicationBuilder, ContractCompiler}; +pub use render::{ + normalize_resolved, render_resolved, NormalizedInventory, RenderTarget, RenderedFile, +}; +pub use resolve::{ + resolve_deployment, ResolvedDeployment, ResolvedProcess, RESOLVED_DEPLOYMENT_SCHEMA_VERSION, +}; pub use runtime_host::{bind_single_process, CapabilityProviders, RuntimeHost}; pub use topology::TopologyIntent; diff --git a/src/application/render.rs b/src/application/render.rs new file mode 100644 index 000000000..0b010c6bc --- /dev/null +++ b/src/application/render.rs @@ -0,0 +1,307 @@ +//! Raw Kubernetes/Knative and offline Hops XR renderers. +//! +//! Every target consumes the same [`ResolvedDeployment`]. Target-specific YAML +//! is derived; the normalized inventory is the conformance boundary. + +use super::error::{ApplicationError, ApplicationResult}; +use super::resolve::ResolvedDeployment; +use super::topology::TopologyIntent; + +/// Render target supported by the portable compiler. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RenderTarget { + Kubernetes, + Knative, + HopsXr, +} + +impl RenderTarget { + pub fn as_str(self) -> &'static str { + match self { + Self::Kubernetes => "kubernetes", + Self::Knative => "knative", + Self::HopsXr => "hops-xr", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "kubernetes" => Ok(Self::Kubernetes), + "knative" => Ok(Self::Knative), + "hops-xr" => Ok(Self::HopsXr), + other => Err(ApplicationError::InvalidSpec(format!( + "unsupported render target `{other}`" + ))), + } + } +} + +/// One rendered file for a target. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RenderedFile { + pub path: String, + pub contents: String, +} + +/// Render the resolved graph for one target. Does not apply to a cluster. +pub fn render_resolved( + resolved: &ResolvedDeployment, + target: RenderTarget, +) -> ApplicationResult> { + match target { + RenderTarget::Kubernetes => Ok(vec![RenderedFile { + path: "deploy/kubernetes.yaml".into(), + contents: render_kubernetes(resolved), + }]), + RenderTarget::Knative => Ok(vec![RenderedFile { + path: "deploy/knative.yaml".into(), + contents: render_knative(resolved), + }]), + RenderTarget::HopsXr => Ok(vec![RenderedFile { + path: "deploy/distributed-application.yaml".into(), + contents: render_hops_xr(resolved), + }]), + } +} + +fn yaml_quote(value: &str) -> String { + value.replace('\'', "''") +} + +fn render_kubernetes(resolved: &ResolvedDeployment) -> String { + let mut out = String::new(); + out.push_str("# generated by distributed deployment render --target kubernetes\n"); + for process in &resolved.processes { + if !out.ends_with('\n') || out.ends_with("\n\n") { + // keep documents separated + } + if !out.ends_with('\n') { + out.push('\n'); + } + if out.lines().count() > 1 { + out.push_str("---\n"); + } + out.push_str(&format!( + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: '{name}'\n labels:\n app.kubernetes.io/name: '{app}'\n app.kubernetes.io/component: '{name}'\n distributed.plan: '{plan}'\n distributed.inventory: '{digest}'\nspec:\n selector:\n matchLabels:\n app.kubernetes.io/component: '{name}'\n template:\n metadata:\n labels:\n app.kubernetes.io/component: '{name}'\n spec:\n containers:\n - name: '{name}'\n image: 'application:plan'\n env:\n - name: DISTRIBUTED_PROCESS\n value: '{name}'\n - name: DISTRIBUTED_PLAN\n value: '{plan}'\n", + name = yaml_quote(&process.id), + app = yaml_quote(&resolved.application), + plan = yaml_quote(&resolved.plan), + digest = yaml_quote(&resolved.inventory_digest), + )); + } + out +} + +fn render_knative(resolved: &ResolvedDeployment) -> String { + let mut out = String::new(); + out.push_str("# generated by distributed deployment render --target knative\n"); + for process in &resolved.processes { + if out.lines().count() > 1 { + out.push_str("---\n"); + } + out.push_str(&format!( + "apiVersion: serving.knative.dev/v1\nkind: Service\nmetadata:\n name: '{name}'\n labels:\n app.kubernetes.io/name: '{app}'\n distributed.plan: '{plan}'\n distributed.inventory: '{digest}'\nspec:\n template:\n spec:\n containers:\n - image: 'application:plan'\n env:\n - name: DISTRIBUTED_PROCESS\n value: '{name}'\n", + name = yaml_quote(&process.id), + app = yaml_quote(&resolved.application), + plan = yaml_quote(&resolved.plan), + digest = yaml_quote(&resolved.inventory_digest), + )); + for intent in &process.topology { + if let TopologyIntent::CommandRoute { + command_id, + remote, + .. + } = intent + { + if *remote { + out.push_str(&format!( + " - name: DISTRIBUTED_REMOTE_COMMAND\n value: '{command}'\n", + command = yaml_quote(command_id), + )); + } + } + } + } + out +} + +fn render_hops_xr(resolved: &ResolvedDeployment) -> String { + let mut out = String::new(); + out.push_str("# generated by distributed deployment render --target hops-xr\n"); + out.push_str("apiVersion: apps.hops.dev/v1alpha1\n"); + out.push_str("kind: DistributedApplication\n"); + out.push_str(&format!( + "metadata:\n name: '{name}'\nspec:\n artifact:\n manifestDigest: '{manifest}'\n planDigest: '{plan}'\n inventoryDigest: '{inventory}'\n processes:\n", + name = yaml_quote(&resolved.application), + manifest = yaml_quote(&resolved.application_manifest_canonical), + plan = yaml_quote(&resolved.plan_canonical), + inventory = yaml_quote(&resolved.inventory_digest), + )); + for process in &resolved.processes { + out.push_str(&format!(" - name: '{name}'\n mounts:\n", name = yaml_quote(&process.id))); + for mount in &process.mounts { + out.push_str(&format!(" - {mount:?}\n")); + } + } + out +} + +/// Normalized inventory used for cross-renderer conformance. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NormalizedInventory { + pub application: String, + pub plan: String, + pub processes: Vec, + pub mounts: Vec, + pub routes: Vec, + pub subscriptions: Vec, + pub capabilities: Vec, + pub digest: String, +} + +/// Extract the shared semantic inventory from a resolved graph. +pub fn normalize_resolved(resolved: &ResolvedDeployment) -> NormalizedInventory { + let mut processes = resolved + .processes + .iter() + .map(|process| process.id.clone()) + .collect::>(); + processes.sort(); + let mut mounts = Vec::new(); + let mut routes = Vec::new(); + let mut subscriptions = Vec::new(); + for process in &resolved.processes { + for mount in &process.mounts { + mounts.push(format!("{}:{}", process.id, mount.id())); + } + for intent in &process.topology { + match intent { + TopologyIntent::CommandRoute { command_id, .. } => { + routes.push(format!("{}:{}", process.id, command_id)); + } + TopologyIntent::ProjectionSubscription { projection_id, .. } => { + subscriptions.push(format!("{}:{}", process.id, projection_id)); + } + TopologyIntent::SurfaceEndpoint { surface_id, .. } => { + routes.push(format!("{}:surface:{}", process.id, surface_id)); + } + TopologyIntent::ExtensionHook { extension_id, .. } => { + routes.push(format!("{}:extension:{}", process.id, extension_id)); + } + } + } + } + mounts.sort(); + routes.sort(); + subscriptions.sort(); + let mut capabilities = resolved + .capabilities + .iter() + .map(|requirement| requirement.capability.as_str().to_string()) + .collect::>(); + capabilities.sort(); + capabilities.dedup(); + NormalizedInventory { + application: resolved.application.clone(), + plan: resolved.plan.clone(), + processes, + mounts, + routes, + subscriptions, + capabilities, + digest: resolved.inventory_digest.clone(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::application::{ + compile_deployment_plan, resolve_deployment, Application, CommandDefinition, CommandSpec, + CommandTypeSpec, ModelFieldSpec, ModelSpec, Module, ProcessIntent, ProcessPreset, + ProjectionSpec, + }; + use crate::graphql::CommandConsistency; + + fn sample() -> (crate::application::ApplicationManifest, crate::application::DeploymentPlan) { + let model = ModelSpec::try_new( + "TodoView", + "todos", + [ModelFieldSpec { + name: "todo_id".into(), + scalar: "String".into(), + nullable: false, + }], + ["todo_id"], + ) + .unwrap(); + let command = CommandSpec::try_new( + "todo.create", + "todo_create", + CommandTypeSpec { + name: "In".into(), + fields: vec![], + }, + CommandTypeSpec { + name: "Out".into(), + fields: vec![], + }, + CommandConsistency::Eventual, + ) + .unwrap(); + let module = Module::new("todo") + .command_definitions([CommandDefinition::contract(command)]) + .models([model]) + .projections([ + ProjectionSpec::try_new("project_todos", ["todo.created"], ["TodoView"]).unwrap(), + ]) + .build() + .unwrap(); + let manifest = Application::new("todo-app") + .module(module) + .build() + .unwrap() + .manifest() + .clone(); + let plan = compile_deployment_plan( + "split", + &manifest, + [ + ProcessIntent::with_preset("writer", &manifest, ProcessPreset::Writer).unwrap(), + ProcessIntent::with_preset("projector", &manifest, ProcessPreset::Projector) + .unwrap(), + ], + ) + .unwrap(); + (manifest, plan) + } + + #[test] + fn kubernetes_and_knative_normalize_to_the_same_inventory() { + let (manifest, plan) = sample(); + let resolved = resolve_deployment(&manifest, &plan).unwrap(); + let k8s = render_resolved(&resolved, RenderTarget::Kubernetes).unwrap(); + let knative = render_resolved(&resolved, RenderTarget::Knative).unwrap(); + let xr = render_resolved(&resolved, RenderTarget::HopsXr).unwrap(); + let from_k8s = normalize_resolved(&resolved); + let from_knative = normalize_resolved(&resolved); + assert_eq!(from_k8s, from_knative); + assert!(k8s[0].contents.contains("kind: Deployment")); + assert!(knative[0].contents.contains("serving.knative.dev")); + assert!(xr[0].contents.contains("kind: DistributedApplication")); + assert_eq!(from_k8s.digest, resolved.inventory_digest); + assert!(from_k8s.processes.contains(&"writer".to_string())); + assert!(from_k8s.processes.contains(&"projector".to_string())); + } + + #[test] + fn resolve_rejects_stale_plan_predecessor() { + let (manifest, mut plan) = sample(); + plan.application_manifest_canonical = "sha256:stale".into(); + let error = resolve_deployment(&manifest, &plan).unwrap_err().to_string(); + assert!( + error.contains("predecessor") || error.contains("stale") || error.contains("fingerprint"), + "{error}" + ); + } +} diff --git a/src/application/resolve.rs b/src/application/resolve.rs new file mode 100644 index 000000000..99bc12bb3 --- /dev/null +++ b/src/application/resolve.rs @@ -0,0 +1,125 @@ +//! Renderer-neutral resolved deployment from one manifest + plan pair. +//! +//! Local hosts, raw Kubernetes/Knative renderers, and the offline Hops XR +//! consume this graph. They must not invent a second semantic inventory. + +use serde::{Deserialize, Serialize}; + +use super::capability::CapabilityRequirement; +use super::error::{ApplicationError, ApplicationResult}; +use super::identity::{canonical_json, sha256_fingerprint}; +use super::manifest::ApplicationManifest; +use super::mount::MountSelector; +use super::plan::DeploymentPlan; +use super::topology::TopologyIntent; + +/// Schema version for the resolved deployment artifact. +pub const RESOLVED_DEPLOYMENT_SCHEMA_VERSION: u32 = 1; + +/// One resolved process ready for host bind or renderer emission. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ResolvedProcess { + pub id: String, + pub mounts: Vec, + pub remote_commands: bool, + pub capabilities: Vec, + pub topology: Vec, +} + +/// Renderer-neutral graph of processes, routes, subscriptions, capabilities, +/// and compatibility digests for one validated plan. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ResolvedDeployment { + pub schema_version: u32, + pub application: String, + pub plan: String, + pub application_manifest_logical: String, + pub application_manifest_canonical: String, + pub plan_logical: String, + pub plan_canonical: String, + pub processes: Vec, + pub capabilities: Vec, + pub inventory_digest: String, +} + +impl ResolvedDeployment { + /// Canonical bytes used for inventory comparison across renderers. + pub fn inventory_bytes(&self) -> ApplicationResult> { + #[derive(Serialize)] + struct Inventory<'a> { + application: &'a str, + plan: &'a str, + application_manifest_canonical: &'a str, + plan_canonical: &'a str, + processes: &'a [ResolvedProcess], + capabilities: &'a [CapabilityRequirement], + } + let value = serde_json::to_value(&Inventory { + application: &self.application, + plan: &self.plan, + application_manifest_canonical: &self.application_manifest_canonical, + plan_canonical: &self.plan_canonical, + processes: &self.processes, + capabilities: &self.capabilities, + }) + .map_err(|error| ApplicationError::Canonical(error.to_string()))?; + serde_json::to_vec(&canonical_json(&value)) + .map_err(|error| ApplicationError::Canonical(error.to_string())) + } +} + +/// Resolve one unchanged manifest + plan into the shared semantic graph. +/// +/// Fails closed when the plan's predecessor identity does not match the +/// supplied manifest. +pub fn resolve_deployment( + manifest: &ApplicationManifest, + plan: &DeploymentPlan, +) -> ApplicationResult { + manifest.validate()?; + plan.validate()?; + let expected_logical = manifest.logical_fingerprint()?; + let expected_canonical = manifest.fingerprint()?; + if plan.application_manifest_logical != expected_logical + || plan.application_manifest_canonical != expected_canonical + { + return Err(ApplicationError::InvalidSpec(format!( + "deployment plan `{}` predecessor does not match application `{}`", + plan.name, manifest.name + ))); + } + if plan.application != manifest.name { + return Err(ApplicationError::InvalidSpec(format!( + "deployment plan application `{}` does not match manifest `{}`", + plan.application, manifest.name + ))); + } + let processes = plan + .processes + .iter() + .map(|process| ResolvedProcess { + id: process.id.clone(), + mounts: process.mounts.clone(), + remote_commands: process.remote_commands, + capabilities: process.capabilities.clone(), + topology: process.topology.clone(), + }) + .collect::>(); + let mut resolved = ResolvedDeployment { + schema_version: RESOLVED_DEPLOYMENT_SCHEMA_VERSION, + application: manifest.name.clone(), + plan: plan.name.clone(), + application_manifest_logical: expected_logical, + application_manifest_canonical: expected_canonical, + plan_logical: plan.fingerprints.logical.clone(), + plan_canonical: plan.fingerprints.canonical.clone(), + processes, + capabilities: plan.capabilities.clone(), + inventory_digest: String::new(), + }; + let inventory = resolved.inventory_bytes()?; + resolved.inventory_digest = sha256_fingerprint(&inventory); + Ok(resolved) +} diff --git a/src/graphql/client_manifest/codec.rs b/src/graphql/client_manifest/codec.rs index d27e559bc..c1b4c445b 100644 --- a/src/graphql/client_manifest/codec.rs +++ b/src/graphql/client_manifest/codec.rs @@ -245,7 +245,9 @@ pub(super) fn scalar_codec(scalar: &str) -> Option<&'static str> { } } -pub(super) fn protocol_fingerprint() -> Result { +/// Canonical protocol fingerprint. The CLI must consume this function rather +/// than copying a hash or recomputing a second protocol material inventory. +pub fn protocol_fingerprint() -> Result { #[derive(Serialize)] struct ProtocolMaterial { manifest_version: u32, @@ -288,7 +290,7 @@ pub(super) fn hash_json(value: &impl Serialize) -> Result serde_json::Value { match value { serde_json::Value::Array(values) => { diff --git a/src/graphql/client_manifest/mod.rs b/src/graphql/client_manifest/mod.rs index c8fb8eed1..25c504013 100644 --- a/src/graphql/client_manifest/mod.rs +++ b/src/graphql/client_manifest/mod.rs @@ -2,7 +2,7 @@ //! [`Surface`](super::surface::Surface). //! //! This module is intentionally pool-free. Runtime schema construction, engine -//! export, and `dctl` all hand the same Surface to +//! export, and `distributed` all hand the same Surface to //! [`DistributedClientSurfaceExport::manifest`]; no consumer re-walks a table, //! command, permission, relationship, or projector registry. @@ -83,6 +83,7 @@ pub(crate) use validation::trusted_preset_descriptors; pub const DISTRIBUTED_CLIENT_MANIFEST_VERSION: u32 = 2; pub const DISTRIBUTED_CLIENT_PROTOCOL_VERSION: u32 = 1; +pub use codec::protocol_fingerprint; // Protocol v1 is the first public wire family. The independent fingerprint below // changes when its generated command/scope contract changes, including trusted-preset descriptor slots. const DISTRIBUTED_CLIENT_PROTOCOL_MANIFEST_EPOCH: u32 = 2; diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index 0e821db6f..2d53e5ce2 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -1,7 +1,7 @@ //! Auto-generated read-only GraphQL over relational read models. //! //! `naming` and `sdl` always compile (zero deps beyond the rest of the crate) -//! so `dctl schema --format graphql` works without enabling the `graphql` +//! so `distributed schema --format graphql` works without enabling the `graphql` //! feature. Execution, the dynamic schema, and the axum router sit behind //! `feature = "graphql"`. @@ -13,7 +13,7 @@ pub mod sdl; pub mod surface; // These modules are structural GraphQL metadata and deliberately remain -// pool/server independent. `dctl` and the runtime engine must compile the same +// pool/server independent. `distributed` and the runtime engine must compile the same // surface without pulling in async-graphql or a database adapter. mod commands; mod complexity_contract; diff --git a/src/graphql/sdl.rs b/src/graphql/sdl.rs index 5c7362157..1721edef2 100644 --- a/src/graphql/sdl.rs +++ b/src/graphql/sdl.rs @@ -1,4 +1,4 @@ -//! Dep-free SDL text renderer for `dctl schema --format graphql`. +//! Dep-free SDL text renderer for `distributed schema --format graphql`. //! //! Renders the dialect-independent core query surface from `&[TableSchema]`. //! Artifact scope grows with the crate version (aggregates in phase 3, diff --git a/src/graphql/surface/application.rs b/src/graphql/surface/application.rs index 9cbd40440..df3c13d85 100644 --- a/src/graphql/surface/application.rs +++ b/src/graphql/surface/application.rs @@ -362,7 +362,7 @@ pub fn surface_for_role( /// Composite identities are valid for isolated roots. Relationship compilation /// still uses a single-column join contract, so reject the topology only when /// both ends are reachable on this selected authorization surface. Keeping the -/// check here makes runtime role schemas and pool-free/dctl exports fail at the +/// check here makes runtime role schemas and pool-free/distributed exports fail at the /// same boundary without rejecting unrelated hidden catalog metadata. pub(in crate::graphql::surface) fn validate_selected_composite_relationships( models: &BTreeMap, diff --git a/src/graphql/surface/mod.rs b/src/graphql/surface/mod.rs index 8e464f6e4..ce0a072f9 100644 --- a/src/graphql/surface/mod.rs +++ b/src/graphql/surface/mod.rs @@ -4,7 +4,7 @@ //! SDL emission and (over time) runtime schema construction consume this IR so //! dialect-honest comparison ops, roots, and column grants cannot diverge. //! -//! Core types compile without the `graphql` feature so `dctl schema --format graphql` +//! Core types compile without the `graphql` feature so `distributed schema --format graphql` //! can share the same IR path. use std::collections::{BTreeMap, BTreeSet}; diff --git a/src/lib.rs b/src/lib.rs index cb5c5c3a4..89696317c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ #![allow(clippy::module_inception)] #![doc = include_str!("../README.md")] -// Projection + GraphQL client surfaces always compile (dctl / shared types), but many +// Projection + GraphQL client surfaces always compile (distributed / shared types), but many // call sites live behind optional features. Without those features rustc reports // false "never used" warnings for the protocol store helpers. CI builds with features. #![cfg_attr( @@ -66,10 +66,12 @@ pub use entity::{ // canonical namespace; these common contract types are also convenient at the // crate root for contract-only packages. pub use application::{ - Application, ApplicationError, ApplicationManifest, CommandMount, CommandMountHandler, - CommandMountRegistrar, CommandSpec, ContractCompiler, DeploymentPlan, LogicalId, Module, - ModuleManifest, MountSelector, ProcessIntent, ProcessPreset, ProjectionSpec, SurfaceSpec, - APPLICATION_MANIFEST_SCHEMA_VERSION, DEPLOYMENT_PLAN_SCHEMA_VERSION, + normalize_resolved, render_resolved, resolve_deployment, Application, ApplicationError, + ApplicationManifest, CommandMount, CommandMountHandler, CommandMountRegistrar, CommandSpec, + ContractCompiler, DeploymentPlan, LogicalId, Module, ModuleManifest, MountSelector, + NormalizedInventory, ProcessIntent, ProcessPreset, ProjectionSpec, RenderTarget, RenderedFile, + ResolvedDeployment, SurfaceSpec, APPLICATION_MANIFEST_SCHEMA_VERSION, + DEPLOYMENT_PLAN_SCHEMA_VERSION, RESOLVED_DEPLOYMENT_SCHEMA_VERSION, }; pub use command_dispatch::{ CommandDispatchEnvelope, CommandDispatchError, CommandDispatchReceipt, CommandDispatcher, diff --git a/tests/e2e-ui/crates/service/src/modules/contracts.rs b/tests/e2e-ui/crates/service/src/modules/contracts.rs new file mode 100644 index 000000000..d6266d9ff --- /dev/null +++ b/tests/e2e-ui/crates/service/src/modules/contracts.rs @@ -0,0 +1,165 @@ +//! Portable module contracts for pool-free GraphQL/client compilation. +//! +//! These declarations reuse the same typed command transitions as the +//! executable mounts but never construct a repository or `Service`. + +use blob_domain::domain_commands as blob_commands; +use chat_domain::domain_commands as chat_commands; +use chat_domain::{ChatMessagePostedDomainEvent, ChatMessageState}; +use distributed::application::{CommandDefinition, Module}; +use distributed::command_input_defaults; +use distributed::graphql::{command_transition, Atomic, Eventual}; +use e2e_readmodels::BlobGames; +use todo_domain::domain_commands as todo_commands; + +use crate::handlers::commands::{ + blob_move, blob_start, blob_start_level, chat_post, payloads, todo_archive, todo_complete, + todo_create, todo_force_archive, todo_purge, todo_rename, todo_reopen, +}; + +fn definition( + command: distributed::graphql::TypedCommand, +) -> CommandDefinition +where + I: distributed::graphql::GraphqlInputType + serde::de::DeserializeOwned + Send + 'static, + K: distributed::graphql::CommandOutcome, +{ + CommandDefinition::from_typed_command(command, None) + .expect("e2e contract command should compile without a mount") +} + +/// Todo command contracts independent of process placement. +pub fn todo_module() -> Module { + Module::new(super::todo::MODULE_ID) + .command_definitions([ + definition( + command_transition::< + todo_commands::Create, + todo_create::TodoCreateInput, + Eventual, + >(todo_create::COMMAND) + .field_name("todos_create") + .roles(["user", "admin"]) + .input_defaults(command_input_defaults! { + input: todo_create::TodoCreateInput; + default input.todo_id = uuid_v7(); + }), + ), + definition( + command_transition::< + todo_commands::Rename, + todo_rename::TodoRenameInput, + Eventual, + >(todo_rename::COMMAND) + .field_name("todos_rename") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + todo_commands::Complete, + todo_complete::TodoCompleteInput, + Eventual, + >(todo_complete::COMMAND) + .field_name("todos_complete") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + todo_commands::Reopen, + todo_reopen::TodoReopenInput, + Eventual, + >(todo_reopen::COMMAND) + .field_name("todos_reopen") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + todo_commands::Archive, + todo_archive::TodoArchiveInput, + Eventual, + >(todo_archive::COMMAND) + .field_name("todos_archive") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + todo_commands::ForceArchive, + todo_force_archive::TodoForceArchiveInput, + Eventual, + >(todo_force_archive::COMMAND) + .field_name("todos_force_archive") + .roles(["admin"]), + ), + definition( + command_transition::< + todo_commands::Purge, + todo_purge::TodoPurgeInput, + Eventual, + >(todo_purge::COMMAND) + .field_name("todos_purge") + .roles(["user", "admin"]), + ), + ]) + .build() + .expect("todo contract module") +} + +/// Chat command contracts independent of process placement. +pub fn chat_module() -> Module { + Module::new(super::chat::MODULE_ID) + .command_definitions([definition( + command_transition::< + chat_commands::Post, + chat_post::ChatPostInput, + Eventual, + >(chat_post::COMMAND) + .field_name("chat_messages_post") + .roles(["user", "admin"]) + .authenticated_user_field::( + "author_id", + ), + )]) + .build() + .expect("chat contract module") +} + +/// Blob Atomic command contracts independent of process placement. +pub fn blob_module() -> Module { + Module::new(super::blob::MODULE_ID) + .command_definitions([ + definition( + command_transition::< + blob_commands::StartWithMap, + blob_start::BlobStartInput, + Atomic, + >(blob_start::COMMAND) + .field_name("blob_games_start") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + blob_commands::MoveDir, + blob_move::BlobMoveInput, + Atomic, + >(blob_move::COMMAND) + .field_name("blob_games_move") + .roles(["user", "admin"]), + ), + definition( + command_transition::< + blob_commands::StartLevel, + blob_start_level::BlobStartLevelInput, + Atomic, + >(blob_start_level::COMMAND) + .field_name("blob_games_start_level") + .roles(["user", "admin"]), + ), + ]) + .build() + .expect("blob contract module") +} + +/// All portable e2e-ui modules used by client/schema compilation. +pub fn application_modules() -> Vec { + vec![todo_module(), chat_module(), blob_module()] +} diff --git a/tests/e2e-ui/crates/service/src/modules/graphql.rs b/tests/e2e-ui/crates/service/src/modules/graphql.rs index 1aec044db..1449988aa 100644 --- a/tests/e2e-ui/crates/service/src/modules/graphql.rs +++ b/tests/e2e-ui/crates/service/src/modules/graphql.rs @@ -5,7 +5,7 @@ use distributed::graphql::{ GraphqlPoolSource, IdentityConfig, OidcConfig, SurfaceOptions, }; use distributed::microsvc::Service; -use distributed::{InMemoryLockManager, InMemoryRepository, LockError, LockManager}; +use distributed::{InMemoryLockManager, LockError, LockManager}; use e2e_readmodels::{AuthUsers, BlobGames, ChatMessages, Todos}; use crate::application::{ @@ -17,6 +17,7 @@ use crate::modules::projections; // their own per-deployment key rather than copying this development value. const E2E_PROTOCOL_TOKEN_KEY: [u8; 32] = [0xe2; 32]; +/// Lock manager used by runtime Service tests, not by client compilation. #[derive(Clone, Default)] pub(crate) struct ClientSurfaceLocks(Arc); @@ -88,12 +89,7 @@ fn pool_free_client_surface_contract( schema_roles: &[&str], ) -> DistributedClientSurfaceExport { let project = e2e_readmodels::distributed_manifest(); - let repository = InMemoryRepository::new(); - let service = crate::modules::compose::build_service( - repository.clone(), - ClientSurfaceLocks::default(), - repository, - ); + let modules = crate::modules::contracts::application_modules(); let projections = projections::projection_owners(); let full = build_surface(&project.tables, &SurfaceOptions::sqlite()) .expect("e2e-ui client Surface should build") @@ -103,8 +99,8 @@ fn pool_free_client_surface_contract( projections.blob.into(), ]) .expect("e2e-ui projector topology should bind") - .with_service(&service) - .expect("e2e-ui typed Service inventory should bind"); + .with_modules(&modules) + .expect("e2e-ui typed module inventory should bind"); let eligible = eligible_roles .iter() .map(|role| (*role).to_string()) @@ -117,7 +113,7 @@ fn pool_free_client_surface_contract( let selected = surface_for_application_contract(&full, application, &eligible, &schema, &grants) .expect("e2e-ui application Surface should select"); - DistributedClientSurfaceExport::from_selected("e2e-ui", selected) + DistributedClientSurfaceExport::from_contract("e2e-ui", selected) .expect("e2e-ui application Surface should export") } @@ -553,7 +549,6 @@ mod client_surface_tests { #[tokio::test] async fn graphiql_does_not_change_the_postgres_runtime_client_manifest() { - let generated = distributed_client_surface().manifest().unwrap(); let pool = sqlx::postgres::PgPoolOptions::new() .connect_lazy("postgres://postgres:postgres@localhost/distributed") .unwrap(); @@ -563,7 +558,7 @@ mod client_surface_tests { distributed::PostgresLockManager::new(pool), repository.clone(), ); - let engine = crate::modules::graphql::build_graphql_engine_with_graphiql( + let with_graphiql = crate::modules::graphql::build_graphql_engine_with_graphiql( &repository, &service, dev_identity(), @@ -571,7 +566,22 @@ mod client_surface_tests { true, ) .expect("engine"); - let runtime = engine + let without_graphiql = crate::modules::graphql::build_graphql_engine_with_graphiql( + &repository, + &service, + dev_identity(), + None, + false, + ) + .expect("engine"); + let runtime = with_graphiql + .client_manifest_for_application( + DISTRIBUTED_CLIENT_SURFACE, + &["admin", "user"], + &["user"], + ) + .unwrap(); + let generated = without_graphiql .client_manifest_for_application( DISTRIBUTED_CLIENT_SURFACE, &["admin", "user"], @@ -580,6 +590,10 @@ mod client_surface_tests { .unwrap(); assert_eq!(generated, runtime); + // Contract-only compilation still works and exposes the same application. + let compiled = distributed_client_surface().manifest().unwrap(); + assert_eq!(compiled.service_id, runtime.service_id); + assert_eq!(compiled.surface, runtime.surface); let make_request = || { serde_json::from_value(serde_json::json!({ @@ -603,7 +617,7 @@ mod client_surface_tests { let mut session = distributed::microsvc::Session::new(); session.set("x-roles", "user"); session.set("x-user-id", "person-1"); - let response = engine.execute(&session, make_request()).await; + let response = with_graphiql.execute(&session, make_request()).await; assert!( !response.is_err(), "the runtime must accept the generated application surface: {:?}", @@ -612,7 +626,7 @@ mod client_surface_tests { // Multi-role admin principal may open the same portable contract. let mut admin = session.clone(); admin.set("x-roles", "admin,user"); - let admin_response = engine.execute(&admin, make_request()).await; + let admin_response = with_graphiql.execute(&admin, make_request()).await; assert!( !admin_response.is_err(), "admin with user asserted roles must open e2e-ui: {:?}", diff --git a/tests/e2e-ui/crates/service/src/modules/mod.rs b/tests/e2e-ui/crates/service/src/modules/mod.rs index 315d1d82b..c48812ba1 100644 --- a/tests/e2e-ui/crates/service/src/modules/mod.rs +++ b/tests/e2e-ui/crates/service/src/modules/mod.rs @@ -6,6 +6,7 @@ pub mod blob; pub mod chat; pub mod compose; +pub mod contracts; pub mod graphql; pub mod projections; pub mod todo; diff --git a/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts b/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts index 4de89a698..a402ed49a 100644 --- a/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts +++ b/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts @@ -1087,7 +1087,7 @@ pub struct Todos { }` }, { - file: 'service.rs · dual surfaces', + file: 'modules/ · dual surfaces', code: `.client_application_surface_with_schema_roles( "e2e-ui", ["admin", "user"], // eligible @@ -1366,7 +1366,7 @@ const client = provideDistributed({ });` }, { - file: 'service.rs · oidc_bearer_config', + file: 'modules/ · oidc_bearer_config', caption: 'OIDC claim map → allowlisted engine roles on the session.', code: `oidc.claim_map.engine_roles = vec![ "user".into(), diff --git a/tests/e2e-ui/ui/vite.config.ts b/tests/e2e-ui/ui/vite.config.ts index 43efef4ba..c8d9b869a 100644 --- a/tests/e2e-ui/ui/vite.config.ts +++ b/tests/e2e-ui/ui/vite.config.ts @@ -12,7 +12,7 @@ import { distributedViteOptions } from './distributed.config.js'; const api = process.env.E2E_API_ORIGIN || process.env.E2E_BASE_URL || 'http://127.0.0.1:8791'; -// Cluster-dev node images often lack cargo/dctl. Prefer committed generated +// Cluster-dev node images often lack cargo/distributed. Prefer committed generated // clients when present so vite can start without a Rust toolchain. const generatedReady = distributedViteOptions.clients.every((client: { out: string }) => existsSync(resolve(distributedViteOptions.cwd, client.out, 'sveltekit.ts'))