From d3fdc748bc7da4cfca27118b1137df0c11b5d06a Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 7 Aug 2026 17:37:02 -0400 Subject: [PATCH 01/16] feat(k8s): add namespace-per-workspace support (RFC 0011 Phase 3) Implement three workspace namespace modes for the Kubernetes compute driver: shared (default, preserves current single-namespace behavior), managed (auto-creates/deletes namespaces per workspace), and operator (pre-provisioned namespaces with dynamic discovery via label selector or drop-in allowlist file). Key changes: - WorkspaceMode enum and namespace resolution in driver config - Managed namespace lifecycle with ServiceAccount and OpenShift SCC annotation propagation - Cluster-wide sandbox CR watchers for managed/operator modes - NamespaceValidator (Exact/Prefix/Allowlist) for SA token auth - Workspace-aware credential secret storage - Helm ClusterRole for multi-namespace RBAC - Gateway config, architecture, and reference docs Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 63 ++ crates/openshell-core/src/driver_utils.rs | 3 + .../src/lib.rs | 74 ++- crates/openshell-driver-kubernetes/README.md | 14 +- .../openshell-driver-kubernetes/src/config.rs | 539 ++++++++++++++++++ .../openshell-driver-kubernetes/src/driver.rs | 533 ++++++++++++++--- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../openshell-driver-kubernetes/src/main.rs | 26 +- crates/openshell-server/src/auth/k8s_sa.rs | 256 ++++++--- crates/openshell-server/src/lib.rs | 26 +- deploy/helm/openshell/README.md | 3 + .../helm/openshell/templates/clusterrole.yaml | 52 ++ .../openshell/templates/gateway-config.yaml | 10 + deploy/helm/openshell/templates/role.yaml | 3 + .../helm/openshell/templates/rolebinding.yaml | 3 + deploy/helm/openshell/values.yaml | 14 + docs/reference/gateway-config.mdx | 15 + 17 files changed, 1466 insertions(+), 173 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e4224232dd..b5994eacdd 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -309,5 +309,68 @@ Standalone local deployments start the gateway with a selected runtime such as Docker, Podman, or VM. The CLI can register multiple gateways and switch between them without changing the sandbox architecture. +## Workspace Namespace Modes (Kubernetes) + +The Kubernetes driver maps workspaces to namespaces through the `workspace_mode` +configuration field (`WorkspaceMode` in `crates/openshell-driver-kubernetes/src/config.rs`). +The mode controls namespace resolution, resource naming, sandbox CR watching, SA +token authentication, and RBAC requirements. + +| Mode | Namespace resolution | Resource name | Namespace lifecycle | +|---|---|---|---| +| **Shared** (default) | Single static namespace from config | `{workspace}--{name}` | None | +| **Managed** | `openshell-{gateway_id}-{workspace}` | bare sandbox name | Driver creates and deletes | +| **Operator** | Workspace name maps 1:1 to a pre-provisioned namespace | bare sandbox name | External (platform team) | + +**Shared** renders all sandboxes into one configured namespace. Resource names +embed the workspace prefix for collision avoidance. No namespace lifecycle +management. RBAC uses a namespace-scoped Role. + +**Managed** auto-creates a K8s namespace per workspace on first sandbox create. +Each new namespace receives a ServiceAccount and copies OpenShift SCC UID-range +and supplemental-group annotations from the gateway namespace when present. The +driver deletes the namespace when the last sandbox in it is removed +(`delete_namespace_if_empty`). Requires a non-empty `gateway_id` (validated as a +DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character +limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace +`create`/`delete` and ServiceAccount `create`/`get` permissions. + +**Operator** uses pre-provisioned namespaces discovered through two optional +sources: a K8s label selector (`operator_namespace_label`) and a drop-in +allowlist file (`operator_namespace_file`). At least one must be configured. +The `OperatorNamespaceAllowlist` (`Arc>>`) is populated +at runtime by background watchers and read by the namespace resolver. Sandbox +creation fails closed if the workspace is not in the current allowlist. Platform +teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as +managed mode but without namespace `create`/`delete` or ServiceAccount +permissions. + +### Watching and Querying + +Managed and operator modes set `is_multi_namespace() == true`, which switches +sandbox CR watchers from namespace-scoped `Api::namespaced` to cluster-wide +`Api::all_with`. In managed mode the driver scopes cluster-wide queries with a +`LABEL_GATEWAY_ID` label selector to support multiple gateways on the same +cluster. K8s Events are not watched in cluster-wide mode — the cluster-wide +watcher emits only sandbox CR changes, not platform events. + +### SA Token Authentication + +The gateway's `K8sServiceAccountAuthenticator` adapts its `NamespaceValidator` +per mode (`crates/openshell-server/src/auth/k8s_sa.rs`): + +- **Shared:** `Exact` — accepts only the single configured namespace. +- **Managed:** `Prefix` — accepts any namespace starting with `openshell-{gateway_id}-`. +- **Operator:** `Allowlist` — accepts namespaces present in the dynamic + `BTreeSet` populated by the label/file watchers. Starts empty (fail-closed) + until the first watcher update. + +### Credential Driver Integration + +The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`) +stores secrets in workspace-specific namespaces when `workspace_mode` is managed +or operator. In shared mode, all secrets render into the single configured +namespace. + When runtime infrastructure changes, validate the relevant sandbox e2e path and update the matching driver README if a maintainer-facing constraint changes. diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 79f2eed3aa..be0d8beb36 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -35,6 +35,9 @@ pub const LABEL_SANDBOX_NAMESPACE: &str = "openshell.ai/sandbox-namespace"; /// Container/pod label carrying the sandbox workspace. pub const LABEL_SANDBOX_WORKSPACE: &str = "openshell.ai/sandbox-workspace"; +/// Label carrying the gateway identity on managed namespaces. +pub const LABEL_GATEWAY_ID: &str = "openshell.ai/gateway-id"; + /// Label selector that matches all OpenShell-managed resources which carry a /// sandbox ID label. Used by list and watch operations to exclude foreign /// resources from the same namespace. diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index 65c655be16..1c59401cb1 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -48,10 +48,33 @@ impl CredentialDriverService { } } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum WorkspaceMode { + #[default] + Shared, + Managed, + Operator, +} + #[derive(Debug, Clone, PartialEq, Eq)] struct KubernetesSecretsDriverSettings { namespace: String, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: String, +} + +impl KubernetesSecretsDriverSettings { + fn target_namespace(&self, workspace: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => self.namespace.clone(), + WorkspaceMode::Managed => { + format!("openshell-{}-{}", self.gateway_id, workspace) + } + WorkspaceMode::Operator => workspace.to_string(), + } + } } #[derive(Debug, Clone, Default, serde::Deserialize)] @@ -59,6 +82,8 @@ struct KubernetesSecretsDriverSettings { struct KubernetesSecretsDriverConfig { namespace: Option, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -134,7 +159,10 @@ impl KubernetesSecretsCredentialDriver { credential_key: &str, ) -> Result { let reference = Self::parse_handle(handle, credential_key)?; - if reference.namespace != self.settings.namespace + // In managed/operator modes secrets live in workspace-specific + // namespaces so cross-namespace handles are expected. + if self.settings.workspace_mode == WorkspaceMode::Shared + && reference.namespace != self.settings.namespace && !self.settings.allow_reference_namespace { return Err(Status::permission_denied(format!( @@ -175,7 +203,7 @@ impl KubernetesSecretsCredentialDriver { reference } else { KubernetesSecretReference { - namespace: self.settings.namespace.clone(), + namespace: self.settings.target_namespace(&request.workspace), secret_name: managed_secret_name( &request.workspace, &request.provider_id, @@ -508,6 +536,8 @@ impl KubernetesSecretsDriverSettings { Ok(Self { namespace, allow_reference_namespace: config.allow_reference_namespace, + workspace_mode: config.workspace_mode, + gateway_id: config.gateway_id.unwrap_or_default(), }) } } @@ -842,6 +872,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -865,6 +897,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: true, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -1065,4 +1099,40 @@ mod tests { assert_eq!(err.code(), Code::FailedPrecondition); assert!(err.message().contains("is not managed by OpenShell")); } + + #[test] + fn target_namespace_shared_returns_static_namespace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell"); + assert_eq!(settings.target_namespace("team-b"), "openshell"); + } + + #[test] + fn target_namespace_managed_computes_from_workspace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell-gw1-team-a"); + assert_eq!(settings.target_namespace("team-b"), "openshell-gw1-team-b"); + } + + #[test] + fn target_namespace_operator_uses_workspace_name() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Operator, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "team-a"); + assert_eq!(settings.target_namespace("prod-ns"), "prod-ns"); + } } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d9c0e17fd8..7f82454083 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -3,8 +3,18 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox -custom resources in the configured namespace. It runs in-process with the -gateway server. +custom resources. It runs in-process with the gateway server and supports three +workspace namespace modes via `workspace_mode`: + +- **Shared** (default): All sandboxes render into a single static namespace. + Resource names use `{workspace}--{name}` for collision avoidance. +- **Managed**: The driver auto-creates/deletes a K8s namespace per workspace + (`openshell-{gateway_id}-{workspace_name}`), creates a ServiceAccount in each, + and copies OpenShift SCC annotations from the gateway namespace when present. +- **Operator**: Workspace names map 1:1 to pre-provisioned namespaces discovered + via label selector (`operator_namespace_label`) and/or drop-in allowlist file + (`operator_namespace_file`). Sandbox creation fails closed if the workspace + namespace is not in the current allowlist. ## Runtime Model diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 4ca02bd71c..182d4190f7 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -3,8 +3,13 @@ use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeSet; use std::path::Path; use std::str::FromStr; +use std::sync::{Arc, RwLock}; + +/// Default gateway identity used in managed-mode namespace naming. +pub const DEFAULT_GATEWAY_ID: &str = "openshell"; /// Default Kubernetes namespace for sandbox resources. pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; @@ -88,6 +93,48 @@ impl FromStr for SupervisorTopology { } } +/// How workspaces map to Kubernetes namespaces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkspaceMode { + /// All sandboxes render into a single statically-configured namespace. + /// Resource names use `{workspace}--{name}` for collision avoidance. + #[default] + Shared, + /// The driver creates and deletes K8s namespaces on demand using the + /// convention `openshell-{gateway_id}-{workspace_name}`. + Managed, + /// Sandboxes render into pre-existing K8s namespaces. The driver has no + /// namespace create/delete permissions. Platform teams manage namespaces + /// via their existing tooling. + Operator, +} + +impl std::fmt::Display for WorkspaceMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shared => f.write_str("shared"), + Self::Managed => f.write_str("managed"), + Self::Operator => f.write_str("operator"), + } + } +} + +impl FromStr for WorkspaceMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "shared" => Ok(Self::Shared), + "managed" => Ok(Self::Managed), + "operator" => Ok(Self::Operator), + other => Err(format!( + "unknown workspace mode '{other}'; expected 'shared', 'managed', or 'operator'" + )), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSidecarConfig { @@ -232,7 +279,24 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { + /// How workspaces map to Kubernetes namespaces. `"shared"` (default) + /// renders all sandboxes into `namespace`; `"managed"` creates per-workspace + /// namespaces on demand; `"operator"` uses pre-provisioned namespaces. + pub workspace_mode: WorkspaceMode, + /// Stable gateway identity used in managed-mode namespace naming + /// (`openshell-{gateway_id}-{workspace}`). Propagated from + /// `gateway_jwt.gateway_id`. + pub gateway_id: String, pub namespace: String, + /// K8s label selector for operator-mode namespace discovery (e.g., + /// `"openshell.ai/workspace=true"`). The driver watches namespaces matching + /// this label and builds the allowlist dynamically. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_label: Option, + /// Path to a drop-in JSON file mapping workspace names to namespace names. + /// Hot-reloaded on change. Delivered via `ConfigMap` volume mount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_file: Option, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by /// the gateway's `TokenReview` bootstrap authenticator. pub service_account_name: String, @@ -351,7 +415,11 @@ pub const ANNOTATION_SCC_SUPPLEMENTAL_GROUPS: &str = "openshift.io/sa.scc.supple impl Default for KubernetesComputeConfig { fn default() -> Self { Self { + workspace_mode: WorkspaceMode::default(), + gateway_id: DEFAULT_GATEWAY_ID.to_string(), namespace: DEFAULT_K8S_NAMESPACE.to_string(), + operator_namespace_label: None, + operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), // Default empty so the gateway omits `imagePullPolicy` from pod @@ -593,6 +661,214 @@ impl KubernetesComputeConfig { } Ok(()) } + + /// Resolve the K8s namespace for a workspace. + /// + /// - **Shared:** returns the static `namespace` config field. + /// - **Managed:** computes `openshell-{gateway_id}-{workspace_name}`. + /// - **Operator:** looks up `workspace` in the dynamic allowlist. Fails + /// closed if the workspace is not found. + pub fn namespace_for_workspace( + &self, + workspace: &str, + operator_allowlist: Option<&OperatorNamespaceAllowlist>, + ) -> Result { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(self.namespace.clone()), + WorkspaceMode::Managed => Ok(managed_namespace(&self.gateway_id, workspace)), + WorkspaceMode::Operator => { + let allowlist = + operator_allowlist.ok_or("operator mode requires a namespace allowlist")?; + let namespaces = allowlist.read(); + if namespaces.contains(workspace) { + Ok(workspace.to_string()) + } else { + Err(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + )) + } + } + } + } + + /// Whether the driver operates across multiple namespaces. + #[must_use] + pub fn is_multi_namespace(&self) -> bool { + !matches!(self.workspace_mode, WorkspaceMode::Shared) + } + + /// Compute the K8s resource name for a sandbox. + /// + /// - **Shared:** `{workspace}--{name}` (namespace doesn't provide isolation). + /// - **Managed/Operator:** bare sandbox name (namespace provides isolation). + #[must_use] + pub fn kube_resource_name(&self, workspace: &str, name: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => format!("{workspace}--{name}"), + WorkspaceMode::Managed | WorkspaceMode::Operator => name.to_string(), + } + } + + /// Validate workspace-mode-specific configuration at startup. + pub fn validate_workspace_mode(&self) -> Result<(), String> { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(()), + WorkspaceMode::Managed => { + if self.gateway_id.is_empty() { + return Err("managed workspace mode requires a non-empty gateway_id".into()); + } + if !is_dns_1123_label(&self.gateway_id) { + return Err(format!( + "gateway_id '{}' is not a valid DNS-1123 label", + self.gateway_id + )); + } + // Workspace names can be up to 19 chars (MAX_ROUTABLE_NAME_LEN + // in the server crate). The managed namespace prefix + + // workspace must fit within 63 chars. + let prefix = managed_namespace_prefix(&self.gateway_id); + if prefix.len() + 19 > 63 { + return Err(format!( + "gateway_id '{}' is too long for managed mode; \ + the namespace prefix '{}' ({} chars) plus the \ + maximum workspace name (19 chars) exceeds the \ + 63-char K8s namespace limit", + self.gateway_id, + prefix, + prefix.len() + )); + } + Ok(()) + } + WorkspaceMode::Operator => { + if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() + { + return Err("operator workspace mode requires at least one of \ + operator_namespace_label or operator_namespace_file" + .into()); + } + if let Some(ref label) = self.operator_namespace_label + && label.is_empty() + { + return Err("operator_namespace_label must not be empty when set".into()); + } + if let Some(ref file) = self.operator_namespace_file + && file.is_empty() + { + return Err("operator_namespace_file must not be empty when set".into()); + } + Ok(()) + } + } + } +} + +/// Compute the managed-mode namespace name for a workspace. +#[must_use] +pub fn managed_namespace(gateway_id: &str, workspace: &str) -> String { + format!("openshell-{gateway_id}-{workspace}") +} + +/// The managed-mode namespace prefix used for SA token validation. +#[must_use] +pub fn managed_namespace_prefix(gateway_id: &str) -> String { + format!("openshell-{gateway_id}-") +} + +/// Check whether a string is a valid DNS-1123 label (lowercase alphanumeric +/// and hyphens, 1-63 chars, must start and end with alphanumeric). +#[must_use] +pub fn is_dns_1123_label(s: &str) -> bool { + let len = s.len(); + if len == 0 || len > 63 { + return false; + } + let bytes = s.as_bytes(); + if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() { + return false; + } + if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() { + return false; + } + bytes + .iter() + .all(|&b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Validate that a workspace name produces a valid K8s namespace name in +/// managed mode (combined length <= 63, DNS-1123 compliant). +pub fn validate_managed_namespace_name(gateway_id: &str, workspace: &str) -> Result<(), String> { + let ns = managed_namespace(gateway_id, workspace); + if !is_dns_1123_label(&ns) { + return Err(format!( + "managed namespace '{ns}' (from workspace '{workspace}') is not a valid DNS-1123 label" + )); + } + Ok(()) +} + +/// Thread-safe dynamic allowlist of valid operator-mode namespaces. +/// +/// Backed by an `Arc>>` that is updated by background +/// tasks (label selector watcher, drop-in file watcher) and read by the SA +/// authenticator and namespace resolver. +#[derive(Debug, Clone)] +pub struct OperatorNamespaceAllowlist { + inner: Arc>>, +} + +impl OperatorNamespaceAllowlist { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(BTreeSet::new())), + } + } + + #[must_use] + pub fn from_set(set: BTreeSet) -> Self { + Self { + inner: Arc::new(RwLock::new(set)), + } + } + + /// Replace the entire allowlist (used by background watchers on refresh). + pub fn replace(&self, new_set: BTreeSet) { + let mut guard = self.inner.write().expect("allowlist lock poisoned"); + *guard = new_set; + } + + /// Merge additional namespaces into the allowlist. + pub fn merge(&self, additional: &BTreeSet) { + let mut guard = self.inner.write().expect("allowlist lock poisoned"); + guard.extend(additional.iter().cloned()); + } + + /// Read the current allowlist snapshot. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner.read().expect("allowlist lock poisoned") + } + + /// Check whether a namespace is in the allowlist. + #[must_use] + pub fn contains(&self, namespace: &str) -> bool { + self.inner + .read() + .expect("allowlist lock poisoned") + .contains(namespace) + } + + /// Return a clone of the inner `Arc` for sharing with background tasks. + #[must_use] + pub fn shared(&self) -> Arc>> { + Arc::clone(&self.inner) + } +} + +impl Default for OperatorNamespaceAllowlist { + fn default() -> Self { + Self::new() + } } fn is_dns1123_subdomain(value: &str) -> bool { @@ -1176,6 +1452,46 @@ mod tests { } } + // -- WorkspaceMode tests -- + + #[test] + fn default_workspace_mode_is_shared() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Shared); + } + + #[test] + fn serde_override_workspace_mode_managed() { + let json = serde_json::json!({ "workspace_mode": "managed" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Managed); + } + + #[test] + fn serde_override_workspace_mode_operator() { + let json = serde_json::json!({ "workspace_mode": "operator" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Operator); + } + + #[test] + fn serde_rejects_invalid_workspace_mode() { + let json = serde_json::json!({ "workspace_mode": "invalid" }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + + #[test] + fn workspace_mode_display_roundtrips() { + for mode in [ + WorkspaceMode::Shared, + WorkspaceMode::Managed, + WorkspaceMode::Operator, + ] { + assert_eq!(mode.to_string().parse::().unwrap(), mode); + } + } + #[test] fn upstream_proxy_config_rejects_unsupported_proxy_scheme() { let cfg = KubernetesComputeConfig { @@ -1261,4 +1577,227 @@ mod tests { }; assert!(cfg.validate_upstream_proxy_config().is_ok()); } + + #[test] + fn namespace_for_workspace_shared() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Shared, + namespace: "sandbox-ns".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "sandbox-ns" + ); + assert_eq!( + cfg.namespace_for_workspace("team-b", None).unwrap(), + "sandbox-ns" + ); + } + + #[test] + fn namespace_for_workspace_managed() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "openshell-gw1-team-a" + ); + } + + #[test] + fn namespace_for_workspace_operator() { + let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["prod".to_string()])); + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("prod", Some(&allowlist)) + .unwrap(), + "prod" + ); + assert!( + cfg.namespace_for_workspace("unknown", Some(&allowlist)) + .is_err() + ); + } + + #[test] + fn kube_resource_name_shared_prefixes_workspace() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.kube_resource_name("ws", "box1"), "ws--box1"); + } + + #[test] + fn kube_resource_name_managed_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn kube_resource_name_operator_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn is_multi_namespace() { + assert!(!KubernetesComputeConfig::default().is_multi_namespace()); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + } + + #[test] + fn validate_workspace_mode_shared_always_ok() { + let cfg = KubernetesComputeConfig::default(); + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_managed_requires_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: String::new(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_invalid_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "INVALID".to_string(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_long_gateway_id() { + // prefix = "openshell-{id}-" = 11 + id.len() + // 11 + 34 + 19 = 64 > 63 → rejected + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(34), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("too long for managed mode"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_max_gateway_id() { + // 11 + 33 + 19 = 63 → accepted + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(33), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_requires_discovery() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_operator_accepts_label_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_accepts_file_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_file: Some("/etc/openshell/namespaces.json".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn dns_1123_label_validation() { + assert!(is_dns_1123_label("openshell")); + assert!(is_dns_1123_label("my-gateway-1")); + assert!(is_dns_1123_label("a")); + assert!(!is_dns_1123_label("")); + assert!(!is_dns_1123_label("UPPER")); + assert!(!is_dns_1123_label("-starts-with-dash")); + assert!(!is_dns_1123_label("ends-with-dash-")); + assert!(!is_dns_1123_label("has_underscore")); + assert!(!is_dns_1123_label(&"a".repeat(64))); + } + + #[test] + fn managed_namespace_naming() { + assert_eq!( + managed_namespace("openshell", "default"), + "openshell-openshell-default" + ); + assert_eq!(managed_namespace("gw1", "team-a"), "openshell-gw1-team-a"); + } + + #[test] + fn validate_managed_namespace_name_accepts_valid() { + validate_managed_namespace_name("gw1", "team-a").unwrap(); + } + + #[test] + fn validate_managed_namespace_name_rejects_too_long() { + let long_workspace = "a".repeat(50); + assert!(validate_managed_namespace_name("openshell", &long_workspace).is_err()); + } + + #[test] + fn operator_allowlist_operations() { + let al = OperatorNamespaceAllowlist::new(); + assert!(!al.contains("ns1")); + + al.replace(BTreeSet::from(["ns1".to_string(), "ns2".to_string()])); + assert!(al.contains("ns1")); + assert!(al.contains("ns2")); + assert!(!al.contains("ns3")); + + al.merge(&BTreeSet::from(["ns3".to_string()])); + assert!(al.contains("ns3")); + + al.replace(BTreeSet::new()); + assert!(!al.contains("ns1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0965c3676e..0d3a1dd2c6 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,12 +7,12 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, + SupervisorTopology, WorkspaceMode, managed_namespace, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ - Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Volume, - VolumeMount, + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, ServiceAccount, + Volume, VolumeMount, }; use kube::api::{ Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, @@ -23,8 +23,9 @@ use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, openshell_sandbox_label_selector, + LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, + LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, + openshell_sandbox_label_selector, }; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::progress::{ @@ -464,6 +465,9 @@ impl std::fmt::Debug for KubernetesComputeDriver { impl KubernetesComputeDriver { pub async fn new(config: KubernetesComputeConfig) -> Result { + config + .validate_workspace_mode() + .map_err(KubernetesDriverError::Precondition)?; config .validate_provider_spiffe_workload_api_socket_path() .map_err(KubernetesDriverError::Precondition)?; @@ -525,6 +529,169 @@ impl KubernetesComputeDriver { &self.config.ssh_socket_path } + pub fn workspace_mode(&self) -> WorkspaceMode { + self.config.workspace_mode + } + + /// Ensure the K8s namespace for a workspace exists (managed mode only). + /// + /// Idempotent: returns the namespace name whether it was just created or + /// already existed. Also creates the sandbox `ServiceAccount` in the + /// namespace. + pub async fn ensure_namespace(&self, workspace: &str) -> Result { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + let ns_api: Api = Api::all(self.client.clone()); + + let gateway_ns_api: Api = Api::all(self.client.clone()); + let gateway_ns_annotations = match tokio::time::timeout( + KUBE_API_TIMEOUT, + gateway_ns_api.get(&self.config.namespace), + ) + .await + { + Ok(Ok(ns)) => ns.metadata.annotations.unwrap_or_default(), + _ => BTreeMap::new(), + }; + + let mut labels = BTreeMap::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), self.config.gateway_id.clone()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); + + let mut annotations = BTreeMap::new(); + for key in [ + crate::config::ANNOTATION_SCC_UID_RANGE, + crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, + ] { + if let Some(val) = gateway_ns_annotations.get(key) { + annotations.insert(key.to_string(), val.clone()); + } + } + + let ns = Namespace { + metadata: ObjectMeta { + name: Some(ns_name.clone()), + labels: Some(labels), + annotations: if annotations.is_empty() { + None + } else { + Some(annotations) + }, + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.create(&PostParams::default(), &ns)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "created managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => { + debug!(namespace = %ns_name, "managed namespace already exists"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating namespace {ns_name}" + ))); + } + } + + self.ensure_service_account(&ns_name).await?; + + Ok(ns_name) + } + + async fn ensure_service_account(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + let sa_api: Api = Api::namespaced(self.client.clone(), namespace); + let sa = ServiceAccount { + metadata: ObjectMeta { + name: Some(self.config.service_account_name.clone()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, sa_api.create(&PostParams::default(), &sa)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %namespace, sa = %self.config.service_account_name, "created service account"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => {} + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating service account in {namespace}" + ))); + } + } + + Ok(()) + } + + /// Delete the managed namespace if it contains no sandboxes (managed mode + /// only). Called after sandbox deletion. + pub async fn delete_namespace_if_empty( + &self, + workspace: &str, + ) -> Result<(), KubernetesDriverError> { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + + let sandbox_api_version = self + .supported_sandbox_api_version(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let agent_api = Self::agent_sandbox_api(self.client.clone(), sandbox_api_version, &ns_name); + + let lp = ListParams::default() + .labels(&openshell_sandbox_label_selector()) + .limit(1); + let list = tokio::time::timeout(KUBE_API_TIMEOUT, agent_api.api.list(&lp)) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!("timeout listing sandboxes in {ns_name}")) + })? + .map_err(KubernetesDriverError::from_kube)?; + + if !list.items.is_empty() { + debug!(namespace = %ns_name, "namespace still has sandboxes, skipping delete"); + return Ok(()); + } + + let ns_api: Api = Api::all(self.client.clone()); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + ns_api.delete(&ns_name, &DeleteParams::default()), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "deleted empty managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout deleting namespace {ns_name}" + ))); + } + } + + Ok(()) + } + fn validate_driver_config_for_sandbox( &self, sandbox: &Sandbox, @@ -539,16 +706,70 @@ impl KubernetesComputeDriver { ) } - fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi { + fn agent_sandbox_api( + client: Client, + sandbox_api_version: &str, + namespace: &str, + ) -> AgentSandboxApi { + let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + let api = Api::namespaced_with(client, namespace, &resource); + AgentSandboxApi { api, resource } + } + + fn cluster_wide_sandbox_api(client: Client, sandbox_api_version: &str) -> AgentSandboxApi { let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); - let api = Api::namespaced_with(client, &self.config.namespace, &resource); + let api = Api::all_with(client, &resource); AgentSandboxApi { api, resource } } - async fn supported_agent_sandbox_api(&self, client: Client) -> Result { + async fn supported_agent_sandbox_api( + &self, + client: Client, + namespace: &str, + ) -> Result { let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; - Ok(self.agent_sandbox_api(client, sandbox_api_version)) + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + namespace, + )) + } + + async fn supported_sandbox_api_for_lookup( + &self, + client: Client, + ) -> Result { + let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; + if self.config.is_multi_namespace() { + Ok(Self::cluster_wide_sandbox_api(client, sandbox_api_version)) + } else { + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + &self.config.namespace, + )) + } + } + + fn sandbox_lookup_selector(&self, sandbox_id: &str) -> String { + let mut selector = + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + if self.config.workspace_mode == WorkspaceMode::Managed { + use std::fmt::Write; + write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); + } + selector + } + + fn openshell_sandbox_selector(&self) -> String { + let mut selector = openshell_sandbox_label_selector(); + if self.config.workspace_mode == WorkspaceMode::Managed { + use std::fmt::Write; + write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); + } + selector } async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { @@ -565,7 +786,11 @@ impl KubernetesComputeDriver { client: Client, ) -> Result<&'static str, String> { for sandbox_api_version in SANDBOX_VERSIONS { - let agent_sandbox_api = self.agent_sandbox_api(client.clone(), sandbox_api_version); + let agent_sandbox_api = Self::agent_sandbox_api( + client.clone(), + sandbox_api_version, + &self.config.namespace, + ); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&ListParams::default().limit(1)), @@ -603,39 +828,27 @@ impl KubernetesComputeDriver { )) } - /// Resolve sandbox UID/GID from config or `OpenShift` SCC namespace annotations. - /// - /// Returns `(uid, gid, ns_annotations_map)`: - /// - If `sandbox_uid` is set in config, returns that (with fallback GID) - /// - Otherwise fetches the target namespace and checks for - /// `openshift.io/sa.scc.uid-range` / `openshift.io/sa.scc.supplemental-groups` - /// annotations. - /// - If neither config nor `OpenShift` is found, returns `(1000, 1000, {})` as defaults. - async fn resolve_sandbox_identity(&self) -> (u32, u32, BTreeMap) { - // Explicit config takes priority — skip namespace lookup entirely. + async fn resolve_sandbox_identity_in_namespace( + &self, + namespace: &str, + ) -> (u32, u32, BTreeMap) { if self.config.sandbox_uid.is_some() { let uid = self.config.resolve_sandbox_uid(None); let gid = self.config.resolve_sandbox_gid(uid, None); return (uid, gid, BTreeMap::new()); } - // Try to read namespace annotations for OpenShift SCC. - // Namespace is namespaced so Api::all works (it's cluster-scoped but - // can list all namespaces) and we filter by name, or use Api::namespaced. let ns_api: Api = Api::all(self.client.clone()); - match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(self.config.namespace.as_str())) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(namespace)).await { Ok(Ok(ns)) => { let anns = ns.metadata.annotations.unwrap_or_default(); tracing::info!( - namespace = %self.config.namespace, + namespace = %namespace, uid_range = ?anns.get(crate::config::ANNOTATION_SCC_UID_RANGE), sup_groups = ?anns.get(crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS), "Resolved namespace annotations for sandbox identity" ); let uid = self.config.resolve_sandbox_uid(Some(&anns)); - // Explicit sandbox_gid config wins; SCC annotation only applies when not set. let baseline_gid = self.config.resolve_sandbox_gid(uid, None); let gid = self.config.sandbox_gid.map_or_else( || { @@ -654,7 +867,7 @@ impl KubernetesComputeDriver { } Ok(Err(e)) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, error = %e, "Failed to fetch namespace for SCC annotations, falling back to defaults" ); @@ -664,7 +877,7 @@ impl KubernetesComputeDriver { } Err(_) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, "Namespace fetch timed out, falling back to defaults" ); let uid = DEFAULT_SANDBOX_UID; @@ -689,7 +902,15 @@ impl KubernetesComputeDriver { let _ = self .validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; - validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + match self.config.workspace_mode { + WorkspaceMode::Shared => { + validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + } + WorkspaceMode::Managed | WorkspaceMode::Operator => { + validate_kubernetes_dns1123_label(&sandbox.name, "sandbox name") + .map_err(tonic::Status::invalid_argument)?; + } + } let gpu_requirements = sandbox .spec .as_ref() @@ -710,15 +931,14 @@ impl KubernetesComputeDriver { pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Fetching sandbox from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => list.items.into_iter().next().map_or_else( @@ -727,9 +947,12 @@ impl KubernetesComputeDriver { Ok(None) }, |obj| { - Ok(sandbox_from_object(&self.config.namespace, obj) - .ok() - .map(|(_, s)| s)) + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) }, ), Ok(Err(err)) => { @@ -756,18 +979,19 @@ impl KubernetesComputeDriver { pub async fn list_sandboxes(&self) -> Result, String> { info!( - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Listing sandboxes from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; + let selector = self.openshell_sandbox_selector(); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api .api - .list(&ListParams::default().labels(&openshell_sandbox_label_selector())), + .list(&ListParams::default().labels(&selector)), ) .await { @@ -777,7 +1001,12 @@ impl KubernetesComputeDriver { .into_iter() .filter_map(|obj| { let name = obj.metadata.name.clone().unwrap_or_default(); - match sandbox_from_object(&self.config.namespace, obj) { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + match sandbox_from_object(&ns, obj) { Ok((_, s)) => Some(s), Err(err) => { warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); @@ -795,7 +1024,6 @@ impl KubernetesComputeDriver { } Ok(Err(err)) => { warn!( - namespace = %self.config.namespace, error = %err, "Failed to list sandboxes from Kubernetes" ); @@ -803,7 +1031,6 @@ impl KubernetesComputeDriver { } Err(_elapsed) => { warn!( - namespace = %self.config.namespace, timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out listing sandboxes from Kubernetes" ); @@ -830,21 +1057,32 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::InvalidArgument)?; let name = sandbox.name.as_str(); + let workspace = sandbox.workspace.as_str(); + + let target_namespace = match self.config.workspace_mode { + WorkspaceMode::Shared => self.config.namespace.clone(), + WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, + WorkspaceMode::Operator => workspace.to_string(), + }; + info!( sandbox_id = %sandbox.id, sandbox_name = %name, - namespace = %self.config.namespace, + namespace = %target_namespace, + workspace = %workspace, + workspace_mode = %self.config.workspace_mode, "Creating sandbox in Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_agent_sandbox_api(self.client.clone(), &target_namespace) .await .map_err(KubernetesDriverError::Message)?; // Resolve sandbox UID/GID from config or OpenShift SCC namespace annotations. - let (resolved_user_id, resolved_group_id, ns_annotations) = - self.resolve_sandbox_identity().await; + let (resolved_user_id, resolved_group_id, ns_annotations) = self + .resolve_sandbox_identity_in_namespace(&target_namespace) + .await; let params = SandboxPodParams { default_image: &self.config.default_image, @@ -889,11 +1127,8 @@ impl KubernetesComputeDriver { let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; - let kube_name = kube_resource_name(&sandbox.workspace, name); + let kube_name = self.config.kube_resource_name(workspace, name); let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); - // Copy only the SCC-related annotations onto the Sandbox CR for - // traceability. Copying the full namespace annotation map exposes - // unrelated cluster metadata and can fail with oversized annotations. let mut annotations = sandbox_annotations(sandbox); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, @@ -905,7 +1140,7 @@ impl KubernetesComputeDriver { } obj.metadata = ObjectMeta { name: Some(kube_name), - namespace: Some(self.config.namespace.clone()), + namespace: Some(target_namespace), labels: Some(sandbox_labels(sandbox)), annotations: Some(annotations), ..Default::default() @@ -1085,19 +1320,18 @@ impl KubernetesComputeDriver { pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Deleting sandbox from Kubernetes" ); - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, preconditions) = match tokio::time::timeout( + let (kube_name, obj_namespace, workspace, preconditions) = match tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api.api.list(&lp), + lookup_api.api.list(&lp), ) .await { @@ -1105,11 +1339,22 @@ impl KubernetesComputeDriver { if let Some(obj) = list.items.into_iter().next() { match obj.metadata.name { Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); let pc = Preconditions { uid: obj.metadata.uid, resource_version: obj.metadata.resource_version, }; - (name, pc) + (name, ns, ws, pc) } None => return Ok(false), } @@ -1139,15 +1384,22 @@ impl KubernetesComputeDriver { } }; + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout( - KUBE_API_TIMEOUT, - agent_sandbox_api.api.delete(&kube_name, &dp), - ) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, "Sandbox deleted from Kubernetes"); + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + if self.config.workspace_mode == WorkspaceMode::Managed + && let Err(e) = self.delete_namespace_if_empty(&workspace).await + { + warn!( + workspace = %workspace, + error = %e, + "Failed to clean up empty managed namespace after sandbox deletion" + ); + } Ok(true) } Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { @@ -1178,10 +1430,9 @@ impl KubernetesComputeDriver { pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => Ok(!list.items.is_empty()), @@ -1196,9 +1447,17 @@ impl KubernetesComputeDriver { // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + if self.config.is_multi_namespace() { + self.watch_sandboxes_cluster_wide().await + } else { + self.watch_sandboxes_single_namespace().await + } + } + + async fn watch_sandboxes_single_namespace(&self) -> Result { let namespace = self.config.namespace.clone(); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.watch_client.clone()) + .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) .await?; let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); let watcher_config = watcher::Config::default().labels(&openshell_sandbox_label_selector()); @@ -1306,6 +1565,85 @@ impl KubernetesComputeDriver { Ok(Box::pin(ReceiverStream::new(rx))) } + + async fn watch_sandboxes_cluster_wide(&self) -> Result { + let sandbox_api_version = self + .supported_sandbox_api_version(self.watch_client.clone()) + .await?; + let cluster_api = + Self::cluster_wide_sandbox_api(self.watch_client.clone(), sandbox_api_version); + let selector = self.openshell_sandbox_selector(); + let watcher_config = watcher::Config::default().labels(&selector); + let mut sandbox_stream = watcher::watcher(cluster_api.api, watcher_config).boxed(); + let (tx, rx) = mpsc::channel(256); + let default_namespace = self.config.namespace.clone(); + + tokio::spawn(async move { + loop { + tokio::select! { + result = sandbox_stream.try_next() => match result { + Ok(Some(Event::Applied(obj))) => { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Deleted(obj))) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Restarted(objs))) => { + for obj in objs { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + } + } + Ok(None) => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "sandbox watcher stream ended unexpectedly".to_string() + ))).await; + break; + } + Err(err) => { + let _ = tx.send(Err(KubernetesDriverError::Message(err.to_string()))).await; + break; + } + }, + () = tx.closed() => break, + } + } + }); + + Ok(Box::pin(ReceiverStream::new(rx))) + } } fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { @@ -1324,10 +1662,6 @@ fn validate_gpu_request( Ok(()) } -fn kube_resource_name(workspace: &str, name: &str) -> String { - format!("{workspace}--{name}") -} - const MAX_KUBE_NAME_LEN: usize = 63; fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), tonic::Status> { @@ -6348,22 +6682,6 @@ mod tests { assert!(validate_kubernetes_dns1123_label("dotted.name", "sandbox name").is_err()); } - #[test] - fn kube_resource_name_qualifies_with_workspace() { - assert_eq!(kube_resource_name("alpha", "work"), "alpha--work"); - assert_eq!( - kube_resource_name("default", "my-sandbox"), - "default--my-sandbox" - ); - } - - #[test] - fn kube_resource_name_different_workspaces_produce_different_names() { - let alpha = kube_resource_name("alpha", "work"); - let beta = kube_resource_name("beta", "work"); - assert_ne!(alpha, beta); - } - #[test] fn kube_resource_name_length_validation_accepts_short_names() { validate_kube_resource_name_length("default", "my-sandbox").unwrap(); @@ -6460,6 +6778,37 @@ mod tests { assert!(result.unwrap_err().contains("not managed by openshell")); } + #[test] + fn sandbox_from_object_uses_object_namespace_over_fallback() { + let obj = DynamicObject { + types: None, + metadata: ObjectMeta { + name: Some("work".to_string()), + namespace: Some("openshell-gw1-team-a".to_string()), + annotations: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ])), + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + ])), + ..Default::default() + }, + data: serde_json::json!({}), + }; + + let (_, sandbox) = sandbox_from_object("openshell", obj).unwrap(); + assert_eq!(sandbox.namespace, "openshell-gw1-team-a"); + assert_eq!(sandbox.workspace, "team-a"); + } + #[test] fn sandbox_from_object_warns_on_managed_cr_missing_workspace() { let obj = DynamicObject { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 7c56c8de5b..d18a23a618 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,9 +6,10 @@ pub mod driver; pub mod grpc; pub use config::{ - AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + OperatorNamespaceAllowlist, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 99df4ea165..15acdec59a 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,9 +10,9 @@ use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, + KubernetesSidecarConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -30,9 +30,25 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, + #[arg(long, env = "OPENSHELL_WORKSPACE_MODE", default_value = "shared")] + workspace_mode: WorkspaceMode, + + #[arg( + long, + env = "OPENSHELL_GATEWAY_ID", + default_value = DEFAULT_GATEWAY_ID + )] + gateway_id: String, + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] sandbox_namespace: String, + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_LABEL")] + operator_namespace_label: Option, + + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_FILE")] + operator_namespace_file: Option, + #[arg( long, env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", @@ -158,7 +174,11 @@ async fn main() -> Result<()> { .init(); let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { + workspace_mode: args.workspace_mode, + gateway_id: args.gateway_id, namespace: args.sandbox_namespace, + operator_namespace_label: args.operator_namespace_label, + operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index eed0e5f083..54f7ed9afa 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,7 +26,8 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use std::sync::Arc; +use std::collections::BTreeSet; +use std::sync::{Arc, RwLock}; use tonic::Status; use tracing::{debug, info, warn}; @@ -135,8 +136,32 @@ impl Authenticator for K8sServiceAccountAuthenticator { } } +/// Validates the namespace extracted from an SA token username against the +/// expected set for the active workspace mode. +#[derive(Debug, Clone)] +pub enum NamespaceValidator { + /// Shared mode: accept only the single configured namespace. + Exact(String), + /// Managed mode: accept any namespace with the managed prefix + /// (`openshell-{gateway_id}-`). + Prefix(String), + /// Operator mode: accept namespaces in the dynamic allowlist. + Allowlist(Arc>>), +} + +impl NamespaceValidator { + pub fn accepts(&self, namespace: &str) -> bool { + match self { + Self::Exact(expected) => namespace == expected, + Self::Prefix(prefix) => namespace.starts_with(prefix.as_str()), + Self::Allowlist(set) => set.read().is_ok_and(|s| s.contains(namespace)), + } + } +} + #[derive(Debug)] struct TokenReviewIdentity { + namespace: String, pod_name: String, pod_uid: String, } @@ -151,59 +176,53 @@ struct SandboxOwnerReference { /// Resolver backed by the apiserver's `TokenReview` API and `kube::Client` /// for the per-pod annotation lookup. pub struct LiveK8sResolver { + client: kube::Client, token_reviews_api: Api, - pods_api: Api, - sandboxes_api_v1beta1: Api, - sandboxes_api_v1alpha1: Api, expected_audience: String, - sandbox_namespace: String, + namespace_validator: NamespaceValidator, expected_service_account: String, } impl LiveK8sResolver { pub fn new( client: kube::Client, - namespace: &str, + namespace_validator: NamespaceValidator, expected_audience: String, expected_service_account: String, ) -> Self { let token_reviews_api: Api = Api::all(client.clone()); - let pods_api: Api = Api::namespaced(client.clone(), namespace); - let sandbox_gvk_v1beta1 = - GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION_V1BETA1, SANDBOX_KIND); - let sandbox_resource_v1beta1 = ApiResource::from_gvk(&sandbox_gvk_v1beta1); - let sandbox_gvk_v1alpha1 = GroupVersionKind::gvk( - SANDBOX_API_GROUP, - SANDBOX_API_VERSION_V1ALPHA1, - SANDBOX_KIND, - ); - let sandbox_resource_v1alpha1 = ApiResource::from_gvk(&sandbox_gvk_v1alpha1); - let sandboxes_api_v1beta1: Api = - Api::namespaced_with(client.clone(), namespace, &sandbox_resource_v1beta1); - let sandboxes_api_v1alpha1: Api = - Api::namespaced_with(client, namespace, &sandbox_resource_v1alpha1); Self { + client, token_reviews_api, - pods_api, - sandboxes_api_v1beta1, - sandboxes_api_v1alpha1, expected_audience, - sandbox_namespace: namespace.to_string(), + namespace_validator, expected_service_account, } } + fn pods_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + fn sandboxes_api(&self, namespace: &str, api_version: &str) -> Api { + let gvk = GroupVersionKind::gvk(SANDBOX_API_GROUP, api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + Api::namespaced_with(self.client.clone(), namespace, &resource) + } + async fn get_sandbox_cr_for_owner( &self, + namespace: &str, owner: &SandboxOwnerReference, ) -> Result, KubeError> { - let apis = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { - [&self.sandboxes_api_v1alpha1, &self.sandboxes_api_v1beta1] + let versions = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { + [SANDBOX_API_VERSION_V1ALPHA1, SANDBOX_API_VERSION_V1BETA1] } else { - [&self.sandboxes_api_v1beta1, &self.sandboxes_api_v1alpha1] + [SANDBOX_API_VERSION_V1BETA1, SANDBOX_API_VERSION_V1ALPHA1] }; - for api in apis { + for version in versions { + let api = self.sandboxes_api(namespace, version); match api.get_opt(&owner.name).await { Ok(Some(sandbox_cr)) => return Ok(Some(sandbox_cr)), Ok(None) => {} @@ -242,7 +261,7 @@ impl K8sIdentityResolver for LiveK8sResolver { let Some(identity) = token_review_identity( &status, &self.expected_audience, - &self.sandbox_namespace, + &self.namespace_validator, &self.expected_service_account, )? else { @@ -252,34 +271,30 @@ impl K8sIdentityResolver for LiveK8sResolver { info!( pod_name = %identity.pod_name, pod_uid = %identity.pod_uid, + namespace = %identity.namespace, service_account = %self.expected_service_account, "validated K8s SA token via TokenReview" ); - // Look up the pod and read its sandbox-id annotation. - let pod = self - .pods_api - .get_opt(&identity.pod_name) - .await - .map_err(|e| { - warn!( - pod = %identity.pod_name, - error = %e, - "failed to fetch sandbox pod for annotation lookup" - ); - Status::internal(format!("pod GET failed: {e}")) - })?; + let pods_api = self.pods_api(&identity.namespace); + let pod = pods_api.get_opt(&identity.pod_name).await.map_err(|e| { + warn!( + pod = %identity.pod_name, + namespace = %identity.namespace, + error = %e, + "failed to fetch sandbox pod for annotation lookup" + ); + Status::internal(format!("pod GET failed: {e}")) + })?; let Some(pod) = pod else { warn!( pod = %identity.pod_name, - "sandbox pod referenced by SA token not found in this namespace" + namespace = %identity.namespace, + "sandbox pod referenced by SA token not found" ); return Err(Status::not_found("sandbox pod not found")); }; - // Defense-in-depth: confirm the pod UID matches the SA token's - // `kubernetes.io.pod.uid`. Prevents a replayed token from a - // recreated pod with the same name. let actual_uid = pod.metadata.uid.as_deref().unwrap_or_default(); if actual_uid != identity.pod_uid { warn!( @@ -294,16 +309,19 @@ impl K8sIdentityResolver for LiveK8sResolver { let sandbox_id = pod_sandbox_id(&pod)?; let owner = sandbox_owner_reference(&pod)?; - let sandbox_cr = self.get_sandbox_cr_for_owner(&owner).await.map_err(|e| { - warn!( - pod = %identity.pod_name, - sandbox_owner = %owner.name, - sandbox_owner_api_version = %owner.api_version, - error = %e, - "failed to fetch owning Sandbox CR for pod identity validation" - ); - Status::internal(format!("sandbox GET failed: {e}")) - })?; + let sandbox_cr = self + .get_sandbox_cr_for_owner(&identity.namespace, &owner) + .await + .map_err(|e| { + warn!( + pod = %identity.pod_name, + sandbox_owner = %owner.name, + sandbox_owner_api_version = %owner.api_version, + error = %e, + "failed to fetch owning Sandbox CR for pod identity validation" + ); + Status::internal(format!("sandbox GET failed: {e}")) + })?; let Some(sandbox_cr) = sandbox_cr else { warn!( pod = %identity.pod_name, @@ -327,7 +345,7 @@ impl K8sIdentityResolver for LiveK8sResolver { fn token_review_identity( status: &TokenReviewStatus, expected_audience: &str, - sandbox_namespace: &str, + namespace_validator: &NamespaceValidator, expected_service_account: &str, ) -> Result, Status> { if status.authenticated != Some(true) { @@ -356,13 +374,20 @@ fn token_review_identity( .username .as_deref() .ok_or_else(|| Status::permission_denied("TokenReview response missing username"))?; - let expected_username = - format!("system:serviceaccount:{sandbox_namespace}:{expected_service_account}"); - if username != expected_username { + + let (namespace, sa_name) = parse_sa_username(username).ok_or_else(|| { warn!( username = %username, - sandbox_namespace = %sandbox_namespace, - service_account = %expected_service_account, + "K8s TokenReview username is not a service account" + ); + Status::permission_denied("SA token username format not recognized") + })?; + + if sa_name != expected_service_account { + warn!( + username = %username, + service_account = %sa_name, + expected = %expected_service_account, "K8s TokenReview principal is not the configured sandbox service account" ); return Err(Status::permission_denied( @@ -370,9 +395,33 @@ fn token_review_identity( )); } + if !namespace_validator.accepts(&namespace) { + warn!( + username = %username, + namespace = %namespace, + "K8s TokenReview SA namespace not accepted by workspace mode validator" + ); + return Err(Status::permission_denied( + "SA token is not from an accepted sandbox namespace", + )); + } + let pod_name = user_extra_one(user, POD_NAME_EXTRA)?; let pod_uid = user_extra_one(user, POD_UID_EXTRA)?; - Ok(Some(TokenReviewIdentity { pod_name, pod_uid })) + Ok(Some(TokenReviewIdentity { + namespace, + pod_name, + pod_uid, + })) +} + +fn parse_sa_username(username: &str) -> Option<(String, String)> { + let rest = username.strip_prefix("system:serviceaccount:")?; + let (namespace, sa_name) = rest.split_once(':')?; + if namespace.is_empty() || sa_name.is_empty() { + return None; + } + Some((namespace.to_string(), sa_name.to_string())) } #[allow(clippy::result_large_err)] @@ -664,6 +713,10 @@ mod tests { cr } + fn exact_validator(ns: &str) -> NamespaceValidator { + NamespaceValidator::Exact(ns.to_string()) + } + #[test] fn token_review_identity_extracts_pod_binding() { let status = token_review_status( @@ -676,10 +729,12 @@ mod tests { ], ); - let identity = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let validator = exact_validator("openshell"); + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .expect("authenticated token should resolve"); + assert_eq!(identity.namespace, "openshell"); assert_eq!(identity.pod_name, "openshell-sandbox-a"); assert_eq!(identity.pod_uid, "uid-a"); } @@ -691,9 +746,10 @@ mod tests { error: Some("invalid audience".to_string()), ..Default::default() }; + let validator = exact_validator("openshell"); assert!( - token_review_identity(&status, "openshell-gateway", "openshell", "default") + token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .is_none() ); @@ -710,8 +766,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("wrong audience must fail closed"); assert_eq!(err.code(), tonic::Code::Unauthenticated); } @@ -727,8 +784,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other namespace must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -744,8 +802,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other service account must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -758,12 +817,71 @@ mod tests { "system:serviceaccount:openshell:default", vec![], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("non pod-bound tokens must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } + #[test] + fn namespace_validator_exact_accepts_matching() { + let v = NamespaceValidator::Exact("openshell".to_string()); + assert!(v.accepts("openshell")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_prefix_accepts_managed_namespaces() { + let v = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + assert!(v.accepts("openshell-gw1-workspace-a")); + assert!(v.accepts("openshell-gw1-default")); + assert!(!v.accepts("openshell-gw2-workspace-a")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_allowlist_accepts_known_namespaces() { + let set = Arc::new(RwLock::new(BTreeSet::from([ + "ns-a".to_string(), + "ns-b".to_string(), + ]))); + let v = NamespaceValidator::Allowlist(set); + assert!(v.accepts("ns-a")); + assert!(v.accepts("ns-b")); + assert!(!v.accepts("ns-c")); + } + + #[test] + fn token_review_identity_prefix_validator_accepts_managed_namespace() { + let status = token_review_status( + true, + vec!["openshell-gateway"], + "system:serviceaccount:openshell-gw1-workspace-a:default", + vec![ + (POD_NAME_EXTRA, "openshell-sandbox-a"), + (POD_UID_EXTRA, "uid-a"), + ], + ); + let validator = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") + .unwrap() + .expect("managed namespace token should resolve"); + assert_eq!(identity.namespace, "openshell-gw1-workspace-a"); + } + + #[test] + fn parse_sa_username_extracts_namespace_and_sa() { + let (ns, sa) = parse_sa_username("system:serviceaccount:openshell:default").unwrap(); + assert_eq!(ns, "openshell"); + assert_eq!(sa, "default"); + + assert!(parse_sa_username("system:node:nodename").is_none()); + assert!(parse_sa_username("system:serviceaccount::default").is_none()); + assert!(parse_sa_username("system:serviceaccount:ns:").is_none()); + } + #[test] fn pod_sandbox_id_requires_annotation() { assert_eq!( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 8ecf69a7a1..7a58c69ba0 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -457,13 +457,33 @@ pub(crate) async fn run_server( compute::driver_config::builtin::kubernetes_config_for_k8s_sa_bootstrap( config_file.as_ref(), )?; - let sandbox_namespace = kubernetes_config.namespace; - let sandbox_service_account = kubernetes_config.service_account_name; + let sandbox_namespace = kubernetes_config.namespace.clone(); + let sandbox_service_account = kubernetes_config.service_account_name.clone(); + let namespace_validator = match kubernetes_config.workspace_mode { + openshell_driver_kubernetes::WorkspaceMode::Shared => { + auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) + } + openshell_driver_kubernetes::WorkspaceMode::Managed => { + auth::k8s_sa::NamespaceValidator::Prefix( + openshell_driver_kubernetes::managed_namespace_prefix( + &kubernetes_config.gateway_id, + ), + ) + } + openshell_driver_kubernetes::WorkspaceMode::Operator => { + // The operator allowlist is populated at runtime by the label + // watcher and file watcher. An empty initial set is fail-closed + // until the watcher populates it. + auth::k8s_sa::NamespaceValidator::Allowlist(Arc::new(std::sync::RwLock::new( + std::collections::BTreeSet::new(), + ))) + } + }; match kube::Client::try_default().await { Ok(client) => { let resolver = Arc::new(auth::k8s_sa::LiveK8sResolver::new( client, - &sandbox_namespace, + namespace_validator, "openshell-gateway".to_string(), sandbox_service_account.clone(), )); diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8752b863d9..b073cc2eb9 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -229,6 +229,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.drivers.kubernetes.operatorNamespaceFile | operator mode | `""` | Path to a drop-in JSON file mapping workspace names to namespace names. Hot-reloaded on change. | +| server.drivers.kubernetes.operatorNamespaceLabel | operator mode | `""` | K8s label selector for namespace discovery. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | | server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 073c8835ec..2acfaa2dad 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -25,9 +26,60 @@ rules: - list - watch # Read namespace annotations for OpenShift SCC UID/GID range resolution. + # Managed/operator modes additionally need list+watch for cluster-wide + # namespace discovery. Managed mode needs create+delete for namespace + # lifecycle. - apiGroups: - "" resources: - namespaces verbs: - get + {{- if ne $workspaceMode "shared" }} + - list + - watch + {{- end }} + {{- if eq $workspaceMode "managed" }} + - create + - delete + {{- end }} + {{- if ne $workspaceMode "shared" }} + # Cluster-wide sandbox CRD access for managed/operator workspace modes. + - apiGroups: + - agents.x-k8s.io + resources: + - sandboxes + - sandboxes/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + {{- end }} + {{- if eq $workspaceMode "managed" }} + # ServiceAccount creation in managed namespaces. + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + {{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 40f5748df4..a8a3128e56 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -135,8 +135,16 @@ data: {{- end }} [openshell.drivers.kubernetes] + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} + operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} + {{- end }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} + operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} + {{- end }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} @@ -204,6 +212,8 @@ data: [openshell.credential_drivers.kubernetes-secrets] namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} {{- end }} {{- if .Values.server.credentialDrivers.vault.enabled }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 5ecc4428ad..4ccc5e3d96 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -42,3 +44,4 @@ rules: - pods verbs: - get +{{- end }} diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index e5233f753c..9bf7c73fab 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -16,3 +18,4 @@ subjects: - kind: ServiceAccount name: {{ include "openshell.serviceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 4fbbf4915c..b8a287b43e 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -243,6 +243,20 @@ server: # the field, "RuntimeDefault" to force the runtime default profile, or # "Localhost/profile-name" for an operator-managed localhost profile. appArmorProfile: "Unconfined" + # Kubernetes compute driver settings. + drivers: + kubernetes: + # -- How workspaces map to Kubernetes namespaces. + # "shared" (default): all sandboxes in a single namespace. + # "managed": auto-creates per-workspace namespaces. + # "operator": uses pre-provisioned namespaces. + workspaceMode: "shared" + # -- (operator mode) K8s label selector for namespace discovery. + # The driver watches namespaces matching this label. + operatorNamespaceLabel: "" + # -- (operator mode) Path to a drop-in JSON file mapping workspace + # names to namespace names. Hot-reloaded on change. + operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. disableTls: false diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 230ae77cee..afb6ed6d9f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -423,6 +423,14 @@ client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" # external_server_names = ["gateway.example.com"] [openshell.drivers.kubernetes] +# Workspace isolation mode. "shared" renders all sandboxes into a single +# namespace. "managed" auto-creates a K8s namespace per workspace +# (openshell-{gateway_id}-{workspace}). "operator" maps each workspace to a +# pre-provisioned namespace discovered via label selector or drop-in file. +workspace_mode = "shared" +# Gateway identity used in managed-mode namespace naming. Defaults to the +# gateway JWT gateway_id. Must be a DNS-1123 label. +# gateway_id = "openshell" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" @@ -491,6 +499,13 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 +# Operator-mode namespace discovery. At least one must be set when +# workspace_mode = "operator". Both can be combined. +# operator_namespace_label discovers namespaces matching a K8s label selector. +# operator_namespace_label = "openshell.ai/workspace=true" +# operator_namespace_file reads allowed namespaces from a JSON/YAML file +# (hot-reloaded on change, e.g. via ConfigMap volume mount). +# operator_namespace_file = "/etc/openshell/workspace-namespaces.json" [openshell.drivers.kubernetes.sidecar] # UID used by relaxed long-running network sidecars. Strict process/binary-aware From 170e659397139437112240451812ac289f1401ce Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 13:32:15 -0400 Subject: [PATCH 02/16] test(k8s): add e2e tests for workspace namespace modes Add end-to-end tests for managed and operator workspace modes introduced in RFC 0011 Phase 3. The managed mode tests verify namespace creation with correct labels, ServiceAccount provisioning, sandbox CR placement, and namespace survival with remaining sandboxes. The operator mode tests verify rejection of unlabeled and nonexistent namespaces. The positive operator path (sandbox in labeled namespace) is known to fail due to an RBAC gap and will be addressed separately. Also fixes Helm 4 compatibility: move SPDX license headers inside conditional guards in 8 chart templates to prevent empty comment-only documents, and fix a trailing whitespace trimmer in clusterrole.yaml that concatenated the license header with apiVersion. Adds cleanup sweep in with-kube-gateway.sh to remove managed and operator namespaces before Helm uninstall, and mise tasks for running each mode independently. Signed-off-by: Derek Carr --- .../ci/values-workspace-managed.yaml | 9 + .../ci/values-workspace-operator.yaml | 10 + .../openshell/templates/cert-manager-pki.yaml | 3 +- .../helm/openshell/templates/clusterrole.yaml | 2 +- .../templates/credential-secrets-role.yaml | 3 +- .../credential-secrets-rolebinding.yaml | 3 +- .../helm/openshell/templates/deployment.yaml | 4 +- deploy/helm/openshell/templates/gateway.yaml | 3 +- .../helm/openshell/templates/grpcroute.yaml | 3 +- deploy/helm/openshell/templates/role.yaml | 5 +- .../helm/openshell/templates/rolebinding.yaml | 5 +- e2e/rust/Cargo.toml | 12 + e2e/rust/tests/workspace_namespace_managed.rs | 301 ++++++++++++++++++ .../tests/workspace_namespace_operator.rs | 263 +++++++++++++++ e2e/with-kube-gateway.sh | 16 + tasks/test.toml | 10 + 16 files changed, 633 insertions(+), 19 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-workspace-managed.yaml create mode 100644 deploy/helm/openshell/ci/values-workspace-operator.yaml create mode 100644 e2e/rust/tests/workspace_namespace_managed.rs create mode 100644 e2e/rust/tests/workspace_namespace_operator.rs diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml new file mode 100644 index 0000000000..9b8911fbe7 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in managed workspace mode. +# Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. +server: + drivers: + kubernetes: + workspaceMode: "managed" diff --git a/deploy/helm/openshell/ci/values-workspace-operator.yaml b/deploy/helm/openshell/ci/values-workspace-operator.yaml new file mode 100644 index 0000000000..8d895e4e98 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-operator.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in operator workspace mode. +# Namespaces must be pre-provisioned and labeled before sandbox creation. +server: + drivers: + kubernetes: + workspaceMode: "operator" + operatorNamespaceLabel: "openshell.ai/e2e-operator-workspace=true" diff --git a/deploy/helm/openshell/templates/cert-manager-pki.yaml b/deploy/helm/openshell/templates/cert-manager-pki.yaml index 2cd8aeabbf..838534e62e 100644 --- a/deploy/helm/openshell/templates/cert-manager-pki.yaml +++ b/deploy/helm/openshell/templates/cert-manager-pki.yaml @@ -1,7 +1,6 @@ +{{- if .Values.certManager.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 2acfaa2dad..66102ab751 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index 72f0528cb2..f6187c9acb 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 4274fa6e1a..3a9ee0bddc 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/deploy/helm/openshell/templates/deployment.yaml b/deploy/helm/openshell/templates/deployment.yaml index e937979370..f94900b136 100644 --- a/deploy/helm/openshell/templates/deployment.yaml +++ b/deploy/helm/openshell/templates/deployment.yaml @@ -1,7 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 {{- include "openshell.validateValues" . }} {{- if eq (include "openshell.workloadKind" .) "deployment" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/helm/openshell/templates/gateway.yaml b/deploy/helm/openshell/templates/gateway.yaml index f431ffbbd1..2b78595053 100644 --- a/deploy/helm/openshell/templates/gateway.yaml +++ b/deploy/helm/openshell/templates/gateway.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: diff --git a/deploy/helm/openshell/templates/grpcroute.yaml b/deploy/helm/openshell/templates/grpcroute.yaml index 8fde5458cd..362067fda3 100644 --- a/deploy/helm/openshell/templates/grpcroute.yaml +++ b/deploy/helm/openshell/templates/grpcroute.yaml @@ -1,7 +1,6 @@ +{{- if .Values.grpcRoute.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.grpcRoute.enabled }} apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 4ccc5e3d96..af80989072 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,8 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - {{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} {{- if eq $workspaceMode "shared" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index 9bf7c73fab..381473a58b 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,8 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - {{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} {{- if eq $workspaceMode "shared" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e2bc94d81d..9ff52c73ab 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -29,6 +29,8 @@ e2e-gpu = ["e2e"] e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] +e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] +e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] @@ -139,6 +141,16 @@ name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" required-features = ["e2e-host-gateway"] +[[test]] +name = "workspace_namespace_managed" +path = "tests/workspace_namespace_managed.rs" +required-features = ["e2e-kubernetes-workspace-managed"] + +[[test]] +name = "workspace_namespace_operator" +path = "tests/workspace_namespace_operator.rs" +required-features = ["e2e-kubernetes-workspace-operator"] + [[test]] name = "gpu" path = "tests/gpu.rs" diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs new file mode 100644 index 0000000000..4bfc793a23 --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-managed")] + +//! E2E tests for managed workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "managed"`, which +//! auto-creates a K8s namespace per workspace (`openshell-{gateway_id}-{ws}`) +//! and deletes it when the last sandbox is removed. +//! +//! Namespace cleanup after sandbox deletion is best-effort and depends on +//! controller finalization timing. These tests focus on verifiable behavior: +//! namespace creation, labels, ServiceAccount provisioning, and sandbox CR +//! placement in the correct namespace. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +fn managed_namespace(workspace: &str) -> String { + format!("openshell-openshell-{workspace}") +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_workspace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +struct ManagedCleanup { + workspace: String, + sandboxes: Vec, +} + +impl Drop for ManagedCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let ns = managed_namespace(&self.workspace); + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &ns, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn managed_creates_namespace_with_labels() { + let ws = unique_workspace("mgd"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["mgd-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create a sandbox — this triggers namespace creation. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "mgd-sb", + "--", + "echo", + "managed-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("managed-ok"), + "sandbox output missing expected string: {out}" + ); + + // Verify the managed namespace was created. + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist: {out}"); + + // Verify labels on the namespace. + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "namespace missing managed-by label: {label_out}" + ); + assert!( + label_out.contains("openshell.ai/gateway-id"), + "namespace missing gateway-id label: {label_out}" + ); + + // Verify the ServiceAccount was created in the managed namespace. + let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; + assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); + + // Verify sandbox CR is in the managed namespace (not the gateway namespace). + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns, + "-o", + "name", + ]) + .await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox CR name mismatch: {out}" + ); +} + +#[tokio::test] +async fn managed_namespace_survives_with_remaining_sandboxes() { + let ws = unique_workspace("mgd2"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["sb-a".into(), "sb-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create two sandboxes. + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws, "--name", "sb-a", "--", "echo", "a", + ]) + .await; + assert!(ok, "sandbox sb-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws, "--name", "sb-b", "--", "echo", "b", + ]) + .await; + assert!(ok, "sandbox sb-b create failed: {out}"); + + // Delete first sandbox — namespace should survive because sb-b still exists. + let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; + assert!(ok, "sandbox sb-a delete failed: {out}"); + + // Brief wait, then verify the namespace still exists. + tokio::time::sleep(Duration::from_secs(3)).await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should still exist with sb-b"); + + // Verify sb-b's CR is still in the managed namespace. + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns, + "-o", + "name", + ]) + .await; + assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); + assert!( + out.contains("sb-b"), + "sb-b CR should still be present: {out}" + ); +} + +#[tokio::test] +async fn managed_isolates_workspaces_into_separate_namespaces() { + let ws_a = unique_workspace("iso-a"); + let ws_b = unique_workspace("iso-b"); + let ns_a = managed_namespace(&ws_a); + let ns_b = managed_namespace(&ws_b); + let _cleanup_a = ManagedCleanup { + workspace: ws_a.clone(), + sandboxes: vec!["sb-iso-a".into()], + }; + let _cleanup_b = ManagedCleanup { + workspace: ws_b.clone(), + sandboxes: vec!["sb-iso-b".into()], + }; + + // Create two workspaces with sandboxes. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_a]).await; + assert!(ok, "workspace A create failed: {out}"); + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_b]).await; + assert!(ok, "workspace B create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws_a, "--name", "sb-iso-a", "--", "echo", "a", + ]) + .await; + assert!(ok, "sandbox A create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws_b, "--name", "sb-iso-b", "--", "echo", "b", + ]) + .await; + assert!(ok, "sandbox B create failed: {out}"); + + // Verify each workspace has its own namespace. + assert_ne!(ns_a, ns_b, "namespaces should differ"); + + let (ok, _) = kubectl(&["get", "namespace", &ns_a]).await; + assert!(ok, "namespace {ns_a} should exist"); + let (ok, _) = kubectl(&["get", "namespace", &ns_b]).await; + assert!(ok, "namespace {ns_b} should exist"); + + // Verify sandbox CRs are in the correct namespaces (no cross-contamination). + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns_a, + "-o", + "name", + ]) + .await; + assert!(ok, "failed to list CRs in {ns_a}: {out}"); + assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); + assert!(!out.contains("sb-iso-b"), "sb-iso-b should NOT be in {ns_a}"); + + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns_b, + "-o", + "name", + ]) + .await; + assert!(ok, "failed to list CRs in {ns_b}: {out}"); + assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); + assert!(!out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}"); +} diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs new file mode 100644 index 0000000000..172414e100 --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-operator")] + +//! E2E tests for operator workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "operator"` and +//! `operator_namespace_label = "openshell.ai/e2e-operator-workspace=true"`. +//! Namespaces must be pre-provisioned and labeled before sandbox creation. +//! The gateway discovers valid namespaces via the label selector. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +const OPERATOR_LABEL: &str = "openshell.ai/e2e-operator-workspace=true"; +const SA_NAME: &str = "openshell-sandbox"; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_namespace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +async fn provision_operator_namespace(name: &str) { + let (ok, out) = kubectl(&["create", "namespace", name]).await; + assert!(ok, "failed to create namespace {name}: {out}"); + + let (ok, out) = kubectl(&["label", "namespace", name, OPERATOR_LABEL]).await; + assert!(ok, "failed to label namespace {name}: {out}"); + + let (ok, out) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", name]).await; + assert!(ok, "failed to create SA in {name}: {out}"); +} + +async fn delete_namespace(name: &str) { + let _ = kubectl(&[ + "delete", + "namespace", + name, + "--ignore-not-found", + "--wait=false", + ]) + .await; +} + +struct OperatorCleanup { + workspace: String, + namespace: String, + sandboxes: Vec, +} + +impl Drop for OperatorCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &self.namespace, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn operator_sandbox_in_labeled_namespace() { + let ns = unique_namespace("op"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["op-sb".into()], + }; + + // Pre-provision the namespace with the operator label and ServiceAccount. + provision_operator_namespace(&ns).await; + + // Wait for the gateway's namespace watcher to discover it. + tokio::time::sleep(Duration::from_secs(5)).await; + + // Create a workspace matching the namespace name (operator mode: 1:1 mapping). + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create a sandbox in the workspace. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "op-sb", + "--", + "echo", + "operator-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("operator-ok"), + "sandbox output missing expected string: {out}" + ); + + // Verify the sandbox CR lives in the pre-provisioned namespace. + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!( + out.contains("op-sb"), + "sandbox CR name should be bare 'op-sb', got: {out}" + ); + + // Clean up. + let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_unlabeled_namespace() { + let ns = unique_namespace("opun"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create namespace WITHOUT the operator label. + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to create namespace: {out}"); + + // Create the ServiceAccount (not the label — that's the point). + let (ok, _) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", &ns]).await; + assert!(ok, "failed to create SA"); + + // Create workspace. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail because namespace is not in the allowlist. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for unlabeled namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_nonexistent_namespace() { + let ns = unique_namespace("opne"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create workspace with no matching namespace at all. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for nonexistent namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; +} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index cfc886ffcd..01b6d855c7 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -277,6 +277,22 @@ cleanup() { --ignore-not-found >/dev/null 2>&1 || true fi + # Sweep managed-mode and operator-mode workspace namespaces before + # uninstalling the Helm release (ClusterRole still needed for deletion). + if command -v kubectl >/dev/null 2>&1 && [ -n "${KUBE_CONTEXT}" ]; then + for label in "openshell.ai/managed-by=openshell" \ + "openshell.ai/e2e-operator-workspace=true"; do + ns_list="$(kctl get namespaces -l "${label}" -o name 2>/dev/null || true)" + if [ -n "${ns_list}" ]; then + echo "Cleaning up namespaces with label ${label}..." + echo "${ns_list}" | while read -r ns_ref; do + kctl delete "${ns_ref}" --wait=false --ignore-not-found \ + 2>/dev/null || true + done + fi + done + fi + if [ "${HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v helm >/dev/null 2>&1; then helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \ diff --git a/tasks/test.toml b/tasks/test.toml index d50750e9cb..0462851ed5 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -170,6 +170,16 @@ description = "Run Kubernetes e2e for provider credential storage backed by Kube env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:workspace-managed"] +description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +run = "e2e/rust/e2e-kubernetes.sh" + +["e2e:kubernetes:workspace-operator"] +description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" From 56574dcf5a02b7fe5c523a6adbc34810e2586f47 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 15:04:54 -0400 Subject: [PATCH 03/16] feat(k8s): add operator namespace label watcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spawn a background kube::runtime::watcher in the K8s driver that watches namespaces matching the configured label selector and populates the OperatorNamespaceAllowlist at runtime. The driver owns the allowlist and exposes its Arc so the server can share the same set with the SA token authenticator. create_sandbox now gates pod creation on the allowlist in operator mode — workspaces whose namespace is not yet labeled are rejected at resource render time rather than silently proceeding. Workspace lifecycle itself is unaffected; only sandbox (resource) creation is gated. Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 112 +++++++++++++++++- crates/openshell-server/src/compute/mod.rs | 17 ++- crates/openshell-server/src/lib.rs | 48 +++++--- e2e/rust/tests/workspace_namespace_managed.rs | 97 +++++++-------- 4 files changed, 205 insertions(+), 69 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0d3a1dd2c6..efbc8cb58e 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6,8 +6,8 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, WorkspaceMode, managed_namespace, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ @@ -451,6 +451,7 @@ pub struct KubernetesComputeDriver { watch_client: Client, sandbox_api_version: Arc>, config: KubernetesComputeConfig, + operator_allowlist: Option, } impl std::fmt::Debug for KubernetesComputeDriver { @@ -501,11 +502,26 @@ impl KubernetesComputeDriver { let watch_client = Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?; + let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { + config.operator_namespace_label.as_ref().map(|label| { + let allowlist = OperatorNamespaceAllowlist::new(); + spawn_namespace_label_watcher( + watch_client.clone(), + label.clone(), + allowlist.clone(), + ); + allowlist + }) + } else { + None + }; + Ok(Self { client, watch_client, sandbox_api_version: Arc::new(OnceCell::new()), config, + operator_allowlist, }) } @@ -517,6 +533,10 @@ impl KubernetesComputeDriver { )) } + pub fn operator_allowlist(&self) -> Option<&OperatorNamespaceAllowlist> { + self.operator_allowlist.as_ref() + } + pub fn default_image(&self) -> &str { &self.config.default_image } @@ -1062,7 +1082,16 @@ impl KubernetesComputeDriver { let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, - WorkspaceMode::Operator => workspace.to_string(), + WorkspaceMode::Operator => { + if let Some(ref allowlist) = self.operator_allowlist + && !allowlist.contains(workspace) + { + return Err(KubernetesDriverError::InvalidArgument(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + workspace.to_string() + } }; info!( @@ -3863,6 +3892,83 @@ fn condition_from_value(value: &serde_json::Value) -> Option { }) } +fn spawn_namespace_label_watcher( + client: Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, +) { + let ns_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + + tokio::spawn(async move { + loop { + let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + + loop { + match stream.try_next().await { + Ok(Some(Event::Applied(ns))) => { + if let Some(name) = ns.metadata.name.as_deref() { + let inner = allowlist.shared(); + let mut guard = inner.write().expect("allowlist lock poisoned"); + if guard.insert(name.to_string()) { + let count = guard.len(); + drop(guard); + info!( + namespace = name, + total = count, + "operator namespace added to allowlist" + ); + } + } + } + Ok(Some(Event::Deleted(ns))) => { + if let Some(name) = ns.metadata.name.as_deref() { + let inner = allowlist.shared(); + let mut guard = inner.write().expect("allowlist lock poisoned"); + if guard.remove(name) { + let count = guard.len(); + drop(guard); + info!( + namespace = name, + total = count, + "operator namespace removed from allowlist" + ); + } + } + } + Ok(Some(Event::Restarted(namespaces))) => { + let names: std::collections::BTreeSet = namespaces + .into_iter() + .filter_map(|ns| ns.metadata.name) + .collect(); + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(err) => { + warn!(error = %err, "operator namespace watcher stream error"); + break; + } + } + } + + tokio::time::sleep(Duration::from_secs(2)).await; + } + }); + + info!( + label_selector = %label_selector, + "operator namespace label watcher started" + ); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index bc98670b89..d60690241f 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -53,6 +53,7 @@ use openshell_driver_docker::DockerComputeDriver; #[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, + OperatorNamespaceAllowlist, }; #[cfg(not(target_os = "windows"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; @@ -775,12 +776,21 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - ) -> Result { + ) -> Result< + ( + Self, + Option>>>, + ), + ComputeError, + > { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + let operator_allowlist_arc = driver + .operator_allowlist() + .map(OperatorNamespaceAllowlist::shared); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); - Self::from_driver( + let runtime = Self::from_driver( ComputeDriverKind::Kubernetes.as_str().to_string(), driver, None, @@ -792,7 +802,8 @@ impl ComputeRuntime { tracing_log_bus, supervisor_sessions, ) - .await + .await?; + Ok((runtime, operator_allowlist_arc)) } pub(crate) async fn new_remote_driver( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 7a58c69ba0..764cf9b62f 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -349,7 +349,7 @@ pub(crate) async fn run_server( gateway_tls_enabled: config.tls.is_some(), endpoint_overrides: &config.compute_driver_endpoints, }; - let compute = build_compute_runtime( + let (compute, operator_allowlist) = build_compute_runtime( &config, driver_startup, store.clone(), @@ -471,12 +471,12 @@ pub(crate) async fn run_server( ) } openshell_driver_kubernetes::WorkspaceMode::Operator => { - // The operator allowlist is populated at runtime by the label - // watcher and file watcher. An empty initial set is fail-closed - // until the watcher populates it. - auth::k8s_sa::NamespaceValidator::Allowlist(Arc::new(std::sync::RwLock::new( - std::collections::BTreeSet::new(), - ))) + // Share the driver's allowlist Arc so the SA authenticator and + // the driver's namespace label watcher use the same set. + let allowlist = operator_allowlist.clone().unwrap_or_else(|| { + Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())) + }); + auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) } }; match kube::Client::try_default().await { @@ -874,6 +874,8 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com // Internal wiring helper: each argument is a distinct piece of runtime state // that must be passed through, so the count is justified. #[allow(clippy::too_many_arguments)] +type OperatorAllowlistArc = Option>>>; + async fn build_compute_runtime( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, @@ -882,19 +884,23 @@ async fn build_compute_runtime( sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, -) -> Result { +) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { let driver = configured_compute_driver(config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); - let runtime = match driver { + let (runtime, operator_allowlist) = match driver { #[cfg(target_os = "windows")] - ConfiguredComputeDriver::Builtin(driver) => Err(unsupported_builtin_compute_driver(driver)), + ConfiguredComputeDriver::Builtin(driver) => { + return Err(Error::execution( + unsupported_builtin_compute_driver(driver).to_string(), + )); + } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { warn_if_kubernetes_sandbox_jwt_expiry_disabled(config); let k8s_config = compute::driver_config::builtin::kubernetes_config_from_context(driver_startup)?; - ComputeRuntime::new_kubernetes( + let (rt, allowlist) = ComputeRuntime::new_kubernetes( k8s_config, store, sandbox_index, @@ -903,12 +909,14 @@ async fn build_compute_runtime( supervisor_sessions.clone(), ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, allowlist) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => { let docker_config = compute::driver_config::builtin::docker_config_from_context(driver_startup)?; - ComputeRuntime::new_docker( + let rt = ComputeRuntime::new_docker( config.clone(), docker_config, store, @@ -918,12 +926,14 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { let podman_config = compute::driver_config::builtin::podman_config_from_context(driver_startup)?; - ComputeRuntime::new_podman( + let rt = ComputeRuntime::new_podman( podman_config, store, sandbox_index, @@ -932,6 +942,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { @@ -941,7 +953,7 @@ async fn build_compute_runtime( .file .and_then(|file| file.openshell.gateway.otlp.as_ref()); let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -950,6 +962,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -962,7 +976,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -971,10 +985,12 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } }; - runtime.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + Ok((runtime, operator_allowlist)) } #[derive(Debug, Clone)] diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 4bfc793a23..f18827478b 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -163,20 +163,9 @@ async fn managed_creates_namespace_with_labels() { assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); // Verify sandbox CR is in the managed namespace (not the gateway namespace). - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); - assert!( - out.contains("mgd-sb"), - "sandbox CR name mismatch: {out}" - ); + assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); } #[tokio::test] @@ -193,13 +182,29 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { // Create two sandboxes. let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws, "--name", "sb-a", "--", "echo", "a", + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-a", + "--", + "echo", + "a", ]) .await; assert!(ok, "sandbox sb-a create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws, "--name", "sb-b", "--", "echo", "b", + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-b", + "--", + "echo", + "b", ]) .await; assert!(ok, "sandbox sb-b create failed: {out}"); @@ -215,15 +220,7 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { assert!(ok, "managed namespace {ns} should still exist with sb-b"); // Verify sb-b's CR is still in the managed namespace. - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); assert!( out.contains("sb-b"), @@ -253,13 +250,29 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { assert!(ok, "workspace B create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws_a, "--name", "sb-iso-a", "--", "echo", "a", + "sandbox", + "create", + "--workspace", + &ws_a, + "--name", + "sb-iso-a", + "--", + "echo", + "a", ]) .await; assert!(ok, "sandbox A create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws_b, "--name", "sb-iso-b", "--", "echo", "b", + "sandbox", + "create", + "--workspace", + &ws_b, + "--name", + "sb-iso-b", + "--", + "echo", + "b", ]) .await; assert!(ok, "sandbox B create failed: {out}"); @@ -273,29 +286,19 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { assert!(ok, "namespace {ns_b} should exist"); // Verify sandbox CRs are in the correct namespaces (no cross-contamination). - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns_a, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_a, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_a}: {out}"); assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); - assert!(!out.contains("sb-iso-b"), "sb-iso-b should NOT be in {ns_a}"); - - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns_b, - "-o", - "name", - ]) - .await; + assert!( + !out.contains("sb-iso-b"), + "sb-iso-b should NOT be in {ns_a}" + ); + + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_b, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_b}: {out}"); assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); - assert!(!out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}"); + assert!( + !out.contains("sb-iso-a"), + "sb-iso-a should NOT be in {ns_b}" + ); } From 29c5381bd8006d2b1617aa3b725f0e506bdf0863 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 15:59:29 -0400 Subject: [PATCH 04/16] fix(k8s): harden operator mode and address review findings Close the fail-open gap in operator mode when only operator_namespace_file is configured: the allowlist is now created unconditionally in operator mode (fail-closed from startup). Implement the namespace file watcher using the notify crate, following the TLS hot-reload pattern (parent-directory watch, 1s debounce, ConfigMap symlink-swap safe). The file format is a JSON array of namespace name strings. Additional fixes from the 10-reviewer audit: - Change allowlist rejection from InvalidArgument to FailedPrecondition so callers know the request may succeed later once the namespace is provisioned. - NamespaceValidator::Allowlist now holds the OperatorNamespaceAllowlist newtype instead of a raw Arc>, eliminating silent denial on RwLock poison. - Verify LABEL_MANAGED_BY and LABEL_GATEWAY_ID ownership before deleting a managed namespace. - Replace fixed 5s sleep in operator e2e test with a 30s poll loop. - Add Helm validation for workspaceMode values. - Fix Helm README type column and description for operator fields. - Add insert/remove methods to OperatorNamespaceAllowlist; label watcher now uses them instead of reaching through shared(). - Reject configs with both operator_namespace_label and operator_namespace_file set. Signed-off-by: Derek Carr --- Cargo.lock | 1 + crates/openshell-driver-kubernetes/Cargo.toml | 1 + .../openshell-driver-kubernetes/src/config.rs | 28 ++- .../openshell-driver-kubernetes/src/driver.rs | 201 +++++++++++++++--- crates/openshell-server/src/auth/k8s_sa.rs | 14 +- crates/openshell-server/src/compute/mod.rs | 12 +- crates/openshell-server/src/lib.rs | 8 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/templates/_helpers.tpl | 4 + deploy/helm/openshell/values.yaml | 6 +- .../tests/workspace_namespace_operator.rs | 47 ++-- 11 files changed, 245 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37b9140600..6b1430934c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3777,6 +3777,7 @@ dependencies = [ "kube", "kube-runtime", "miette", + "notify", "openshell-core", "openshell-policy", "prost", diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index f9f5bba398..714b7d05c9 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +notify = "8" [dev-dependencies] temp-env = "0.3" diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 182d4190f7..2df422e588 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -293,8 +293,8 @@ pub struct KubernetesComputeConfig { /// this label and builds the allowlist dynamically. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_label: Option, - /// Path to a drop-in JSON file mapping workspace names to namespace names. - /// Hot-reloaded on change. Delivered via `ConfigMap` volume mount. + /// Path to a JSON file containing an array of namespace names allowed in + /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_file: Option, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by @@ -743,10 +743,16 @@ impl KubernetesComputeConfig { WorkspaceMode::Operator => { if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() { - return Err("operator workspace mode requires at least one of \ + return Err("operator workspace mode requires exactly one of \ operator_namespace_label or operator_namespace_file" .into()); } + if self.operator_namespace_label.is_some() && self.operator_namespace_file.is_some() + { + return Err("operator workspace mode requires exactly one of \ + operator_namespace_label or operator_namespace_file, not both" + .into()); + } if let Some(ref label) = self.operator_namespace_label && label.is_empty() { @@ -858,6 +864,22 @@ impl OperatorNamespaceAllowlist { .contains(namespace) } + /// Insert a namespace into the allowlist. Returns `true` if it was new. + pub fn insert(&self, name: String) -> bool { + self.inner + .write() + .expect("allowlist lock poisoned") + .insert(name) + } + + /// Remove a namespace from the allowlist. Returns `true` if it was present. + pub fn remove(&self, name: &str) -> bool { + self.inner + .write() + .expect("allowlist lock poisoned") + .remove(name) + } + /// Return a clone of the inner `Arc` for sharing with background tasks. #[must_use] pub fn shared(&self) -> Arc>> { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index efbc8cb58e..b117dd303c 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -43,7 +43,7 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -503,15 +503,21 @@ impl KubernetesComputeDriver { Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?; let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { - config.operator_namespace_label.as_ref().map(|label| { - let allowlist = OperatorNamespaceAllowlist::new(); + let allowlist = OperatorNamespaceAllowlist::new(); + + if let Some(ref label) = config.operator_namespace_label { spawn_namespace_label_watcher( watch_client.clone(), label.clone(), allowlist.clone(), ); - allowlist - }) + } + + if let Some(ref path) = config.operator_namespace_file { + spawn_namespace_file_watcher(path.into(), allowlist.clone()); + } + + Some(allowlist) } else { None }; @@ -689,6 +695,36 @@ impl KubernetesComputeDriver { } let ns_api: Api = Api::all(self.client.clone()); + + let ns = match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + return Ok(()); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout getting namespace {ns_name}" + ))); + } + }; + + let labels = ns.metadata.labels.as_ref(); + let is_owned = labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == &self.config.gateway_id); + if !is_owned { + debug!( + namespace = %ns_name, + "namespace not owned by this gateway, skipping delete" + ); + return Ok(()); + } + match tokio::time::timeout( KUBE_API_TIMEOUT, ns_api.delete(&ns_name, &DeleteParams::default()), @@ -1086,7 +1122,7 @@ impl KubernetesComputeDriver { if let Some(ref allowlist) = self.operator_allowlist && !allowlist.contains(workspace) { - return Err(KubernetesDriverError::InvalidArgument(format!( + return Err(KubernetesDriverError::Precondition(format!( "workspace '{workspace}' is not in the operator namespace allowlist" ))); } @@ -3907,33 +3943,20 @@ fn spawn_namespace_label_watcher( loop { match stream.try_next().await { Ok(Some(Event::Applied(ns))) => { - if let Some(name) = ns.metadata.name.as_deref() { - let inner = allowlist.shared(); - let mut guard = inner.write().expect("allowlist lock poisoned"); - if guard.insert(name.to_string()) { - let count = guard.len(); - drop(guard); - info!( - namespace = name, - total = count, - "operator namespace added to allowlist" - ); - } + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); } } Ok(Some(Event::Deleted(ns))) => { - if let Some(name) = ns.metadata.name.as_deref() { - let inner = allowlist.shared(); - let mut guard = inner.write().expect("allowlist lock poisoned"); - if guard.remove(name) { - let count = guard.len(); - drop(guard); - info!( - namespace = name, - total = count, - "operator namespace removed from allowlist" - ); - } + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); } } Ok(Some(Event::Restarted(namespaces))) => { @@ -3965,10 +3988,126 @@ fn spawn_namespace_label_watcher( info!( label_selector = %label_selector, - "operator namespace label watcher started" + "operator namespace label watcher spawned" ); } +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher(path: PathBuf, allowlist: OperatorNamespaceAllowlist) { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + path = %path.display(), + total = count, + "operator namespace allowlist loaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to load initial operator namespace file, allowlist empty" + ); + } + } + + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let debounce = Duration::from_secs(1); + + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut watcher = + match notify::recommended_watcher(move |res: Result| { + if let Ok(event) = res + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }) { + Ok(w) => w, + Err(e) => { + warn!( + error = %e, + "failed to start operator namespace file watcher, hot-reload disabled" + ); + return; + } + }; + + if let Err(e) = notify::Watcher::watch( + &mut watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!( + error = %e, + dir = %watch_dir.display(), + "failed to watch operator namespace file directory, hot-reload disabled" + ); + return; + } + + info!( + path = %path.display(), + "operator namespace file watcher started" + ); + + loop { + let got_event = rx.recv().await.is_some(); + if !got_event { + warn!("operator namespace file watcher disconnected"); + break; + } + + loop { + tokio::select! { + () = tokio::time::sleep(debounce) => { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist reloaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to reload operator namespace file, keeping existing allowlist" + ); + } + } + break; + } + r = rx.recv() => { + if r.is_some() { + continue; + } + warn!("operator namespace file watcher disconnected"); + return; + } + } + } + } + }); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 54f7ed9afa..32cb2e119c 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,8 +26,8 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use std::collections::BTreeSet; -use std::sync::{Arc, RwLock}; +use openshell_driver_kubernetes::OperatorNamespaceAllowlist; +use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; @@ -146,7 +146,7 @@ pub enum NamespaceValidator { /// (`openshell-{gateway_id}-`). Prefix(String), /// Operator mode: accept namespaces in the dynamic allowlist. - Allowlist(Arc>>), + Allowlist(OperatorNamespaceAllowlist), } impl NamespaceValidator { @@ -154,7 +154,7 @@ impl NamespaceValidator { match self { Self::Exact(expected) => namespace == expected, Self::Prefix(prefix) => namespace.starts_with(prefix.as_str()), - Self::Allowlist(set) => set.read().is_ok_and(|s| s.contains(namespace)), + Self::Allowlist(al) => al.contains(namespace), } } } @@ -842,11 +842,11 @@ mod tests { #[test] fn namespace_validator_allowlist_accepts_known_namespaces() { - let set = Arc::new(RwLock::new(BTreeSet::from([ + let al = OperatorNamespaceAllowlist::from_set(std::collections::BTreeSet::from([ "ns-a".to_string(), "ns-b".to_string(), - ]))); - let v = NamespaceValidator::Allowlist(set); + ])); + let v = NamespaceValidator::Allowlist(al); assert!(v.accepts("ns-a")); assert!(v.accepts("ns-b")); assert!(!v.accepts("ns-c")); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index d60690241f..68b8670a4b 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -776,19 +776,11 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - ) -> Result< - ( - Self, - Option>>>, - ), - ComputeError, - > { + ) -> Result<(Self, Option), ComputeError> { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; - let operator_allowlist_arc = driver - .operator_allowlist() - .map(OperatorNamespaceAllowlist::shared); + let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); let runtime = Self::from_driver( ComputeDriverKind::Kubernetes.as_str().to_string(), diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 764cf9b62f..dfc2645d90 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -471,11 +471,7 @@ pub(crate) async fn run_server( ) } openshell_driver_kubernetes::WorkspaceMode::Operator => { - // Share the driver's allowlist Arc so the SA authenticator and - // the driver's namespace label watcher use the same set. - let allowlist = operator_allowlist.clone().unwrap_or_else(|| { - Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())) - }); + let allowlist = operator_allowlist.clone().unwrap_or_default(); auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) } }; @@ -874,7 +870,7 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com // Internal wiring helper: each argument is a distinct piece of runtime state // that must be passed through, so the count is justified. #[allow(clippy::too_many_arguments)] -type OperatorAllowlistArc = Option>>>; +type OperatorAllowlistArc = Option; async fn build_compute_runtime( config: &Config, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index b073cc2eb9..91008ff1d0 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -229,8 +229,8 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | -| server.drivers.kubernetes.operatorNamespaceFile | operator mode | `""` | Path to a drop-in JSON file mapping workspace names to namespace names. Hot-reloaded on change. | -| server.drivers.kubernetes.operatorNamespaceLabel | operator mode | `""` | K8s label selector for namespace discovery. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | +| server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3764fa6d7a..548418abc6 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -247,6 +247,10 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}} {{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}} {{- end -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} +{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} +{{- end -}} {{- $credentialDrivers := list -}} {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} {{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b8a287b43e..0b2fa1098c 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -251,11 +251,11 @@ server: # "managed": auto-creates per-workspace namespaces. # "operator": uses pre-provisioned namespaces. workspaceMode: "shared" - # -- (operator mode) K8s label selector for namespace discovery. + # -- K8s label selector for namespace discovery in operator mode. # The driver watches namespaces matching this label. operatorNamespaceLabel: "" - # -- (operator mode) Path to a drop-in JSON file mapping workspace - # names to namespace names. Hot-reloaded on change. + # -- Path to a JSON file containing an array of namespace names + # allowed in operator mode. Hot-reloaded on change. operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index 172414e100..a382de446c 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -139,30 +139,39 @@ async fn operator_sandbox_in_labeled_namespace() { // Pre-provision the namespace with the operator label and ServiceAccount. provision_operator_namespace(&ns).await; - // Wait for the gateway's namespace watcher to discover it. - tokio::time::sleep(Duration::from_secs(5)).await; - // Create a workspace matching the namespace name (operator mode: 1:1 mapping). let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; assert!(ok, "workspace create failed: {out}"); - // Create a sandbox in the workspace. - let (ok, out) = run_cli(&[ - "sandbox", - "create", - "--workspace", - &ns, - "--name", - "op-sb", - "--", - "echo", - "operator-ok", - ]) - .await; - assert!(ok, "sandbox create failed: {out}"); + // Poll until the gateway's namespace watcher discovers the labeled namespace + // and sandbox creation succeeds (up to 30s). + let mut sandbox_out = String::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "op-sb", + "--", + "echo", + "operator-ok", + ]) + .await; + if ok { + sandbox_out = out; + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } assert!( - out.contains("operator-ok"), - "sandbox output missing expected string: {out}" + sandbox_out.contains("operator-ok"), + "sandbox output missing expected string: {sandbox_out}" ); // Verify the sandbox CR lives in the pre-provisioned namespace. From f6dd2dc45ea73e76c896a618a8084ea07dc22bd3 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Mon, 10 Aug 2026 19:22:02 -0400 Subject: [PATCH 05/16] feat(k8s): add workspace-level compute driver RPCs and harden RBAC Decouple namespace lifecycle from sandbox lifecycle by adding EnsureWorkspace/DeleteWorkspace RPCs to the ComputeDriver service. Namespace creation now happens before credential storage and namespace deletion happens on workspace delete, fixing credential storage in managed workspace mode. - Add EnsureWorkspace and DeleteWorkspace proto RPCs with implementations across all compute drivers (K8s managed delegates to ensure_namespace/delete_namespace_if_empty; others no-op) - Wire ensure_workspace into provider create/update/refresh paths so the namespace exists before the credential driver writes secrets - Wire delete_workspace into workspace deletion for cleanup - Remove delete_namespace_if_empty from sandbox deletion path - Scope ClusterRole secrets access to non-shared workspace modes - Add TODO for TLS cert hot-reload in sandbox gRPC client - Harden e2e tests with control-plane sandbox resolution assertions - Fix docker image save --platform flag for OCI index manifests Signed-off-by: Derek Carr --- crates/openshell-core/src/grpc_client.rs | 4 + crates/openshell-driver-docker/src/lib.rs | 19 ++- .../openshell-driver-kubernetes/src/driver.rs | 152 ++++++++++++++++-- .../openshell-driver-kubernetes/src/grpc.rs | 43 ++++- crates/openshell-driver-podman/src/grpc.rs | 22 ++- crates/openshell-driver-vm/src/driver.rs | 28 +++- crates/openshell-server/src/compute/mod.rs | 103 +++++++++++- crates/openshell-server/src/grpc/provider.rs | 7 + crates/openshell-server/src/grpc/workspace.rs | 4 + .../openshell-server/src/provider_refresh.rs | 57 +++++-- crates/openshell-server/src/test_support.rs | 24 ++- .../helm/openshell/templates/clusterrole.yaml | 22 +++ e2e/rust/tests/workspace_namespace_managed.rs | 62 +++++++ .../tests/workspace_namespace_operator.rs | 25 ++- proto/compute_driver.proto | 21 +++ tasks/scripts/helm-k3s-local.sh | 6 +- 16 files changed, 550 insertions(+), 49 deletions(-) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 1640fd6cf7..80ab0435e1 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -146,6 +146,10 @@ async fn build_plain_channel(endpoint: &str) -> Result { let tls_enabled = endpoint.starts_with("https://"); + // TODO: TLS certs are loaded once here and never re-read. The gateway + // server side supports hot-reload (ArcSwap + notify in tls.rs). The + // supervisor should do the same so that cert-manager rotations take + // effect without restarting the sandbox. if tls_enabled { let ca_path = std::env::var(sandbox_env::TLS_CA) .into_diagnostic() diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 4621852fc9..9ac14c1871 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -38,8 +38,9 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, + DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, @@ -1761,6 +1762,20 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } impl DockerProvisioningFailure { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index b117dd303c..7c6379a093 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -11,8 +11,8 @@ use crate::config::{ }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ - Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, ServiceAccount, - Volume, VolumeMount, + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, + ServiceAccount, Volume, VolumeMount, }; use kube::api::{ Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, @@ -564,6 +564,15 @@ impl KubernetesComputeDriver { /// Idempotent: returns the namespace name whether it was just created or /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. + /// + /// TODO: no `NetworkPolicy` is created here. The Helm-managed namespace gets + /// an SSH-isolation policy (port 2222 restricted to the gateway pod), but + /// managed-mode namespaces do not. Current risk is low: only sandbox pods + /// from the same workspace run in this namespace, so there is no lateral + /// movement target. A same-cluster `namespaceSelector` policy would also + /// break cross-cluster topologies where the gateway is external. Add a + /// configurable `NetworkPolicy` when mixed-workload or cross-cluster managed + /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); let ns_api: Api = Api::all(self.client.clone()); @@ -665,8 +674,108 @@ impl KubernetesComputeDriver { Ok(()) } + /// Ensure the client TLS Secret exists in `namespace` by copying it from + /// the gateway's Helm release namespace. Idempotent: creates the Secret on + /// first call, updates it on subsequent calls to pick up cert rotations. + /// No-op when `client_tls_secret_name` is empty (TLS disabled). + async fn ensure_tls_secret(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + if self.config.client_tls_secret_name.is_empty() { + return Ok(()); + } + + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let source = match tokio::time::timeout( + KUBE_API_TIMEOUT, + source_api.get(&self.config.client_tls_secret_name), + ) + .await + { + Ok(Ok(s)) => s, + Ok(Err(e)) => { + warn!( + secret = %self.config.client_tls_secret_name, + source_namespace = %self.config.namespace, + error = %e, + "failed to read source TLS secret" + ); + return Err(KubernetesDriverError::from_kube(e)); + } + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading TLS secret {} from {}", + self.config.client_tls_secret_name, self.config.namespace + ))); + } + }; + + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + let copy = Secret { + metadata: ObjectMeta { + name: Some(self.config.client_tls_secret_name.clone()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + }; + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.create(&PostParams::default(), ©), + ) + .await + { + Ok(Ok(_)) => { + info!( + namespace = %namespace, + secret = %self.config.client_tls_secret_name, + "created TLS secret copy" + ); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => { + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.replace( + &self.config.client_tls_secret_name, + &PostParams::default(), + ©, + ), + ) + .await + { + Ok(Ok(_)) => { + debug!( + namespace = %namespace, + secret = %self.config.client_tls_secret_name, + "updated TLS secret copy" + ); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout updating TLS secret in {namespace}" + ))); + } + } + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating TLS secret in {namespace}" + ))); + } + } + + Ok(()) + } + /// Delete the managed namespace if it contains no sandboxes (managed mode - /// only). Called after sandbox deletion. + /// only). Called via the `DeleteWorkspace` RPC after workspace deletion. pub async fn delete_namespace_if_empty( &self, workspace: &str, @@ -1130,6 +1239,10 @@ impl KubernetesComputeDriver { } }; + if self.config.is_multi_namespace() { + self.ensure_tls_secret(&target_namespace).await?; + } + info!( sandbox_id = %sandbox.id, sandbox_name = %name, @@ -1206,7 +1319,7 @@ impl KubernetesComputeDriver { obj.metadata = ObjectMeta { name: Some(kube_name), namespace: Some(target_namespace), - labels: Some(sandbox_labels(sandbox)), + labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; @@ -1394,7 +1507,7 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, workspace, preconditions) = match tokio::time::timeout( + let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api.api.list(&lp), ) @@ -1456,15 +1569,6 @@ impl KubernetesComputeDriver { match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - if self.config.workspace_mode == WorkspaceMode::Managed - && let Err(e) = self.delete_namespace_if_empty(&workspace).await - { - warn!( - workspace = %workspace, - error = %e, - "Failed to clean up empty managed namespace after sandbox deletion" - ); - } Ok(true) } Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { @@ -1740,7 +1844,7 @@ fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), Ok(()) } -fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { +fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); @@ -1752,6 +1856,9 @@ fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string(), ); + if let Some(gw_id) = gateway_id { + labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + } labels } @@ -7087,7 +7194,7 @@ mod tests { workspace: "alpha".to_string(), ..Default::default() }; - let labels = sandbox_labels(&sandbox); + let labels = sandbox_labels(&sandbox, None); assert_eq!(labels.get(LABEL_SANDBOX_ID).unwrap(), "uuid-1"); assert_eq!(labels.get(LABEL_SANDBOX_NAME).unwrap(), "work"); assert_eq!(labels.get(LABEL_SANDBOX_WORKSPACE).unwrap(), "alpha"); @@ -7095,6 +7202,19 @@ mod tests { labels.get(LABEL_MANAGED_BY).unwrap(), LABEL_MANAGED_BY_VALUE ); + assert!(!labels.contains_key(LABEL_GATEWAY_ID)); + } + + #[test] + fn sandbox_labels_includes_gateway_id_when_provided() { + let sandbox = Sandbox { + id: "uuid-1".to_string(), + name: "work".to_string(), + workspace: "alpha".to_string(), + ..Default::default() + }; + let labels = sandbox_labels(&sandbox, Some("gw-42")); + assert_eq!(labels.get(LABEL_GATEWAY_ID).unwrap(), "gw-42"); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index ef2e2686e4..fbfcecaac1 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -6,9 +6,11 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -17,6 +19,7 @@ use std::pin::Pin; use tonic::{Request, Response, Status}; use crate::KubernetesComputeDriver; +use crate::WorkspaceMode; #[derive(Debug, Clone)] pub struct ComputeDriverService { @@ -172,6 +175,40 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + if self.driver.workspace_mode() == WorkspaceMode::Managed { + self.driver + .ensure_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + if self.driver.workspace_mode() == WorkspaceMode::Managed { + self.driver + .delete_namespace_if_empty(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(Response::new(DeleteWorkspaceResponse {})) + } } fn kubernetes_lifecycle_status(message: String) -> Status { diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 8d34660514..79081e9419 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -6,9 +6,11 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -170,6 +172,20 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index af914ec467..a61ec11726 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -38,12 +38,14 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, - DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, - DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition as SandboxCondition, + DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, + DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, + EnsureWorkspaceRequest, EnsureWorkspaceResponse, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, @@ -3432,6 +3434,20 @@ impl ComputeDriver for VmDriver { let stream: Self::WatchSandboxesStream = Box::pin(ReceiverStream::new(out_rx)); Ok(Response::new(stream)) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(target_os = "linux")] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 68b8670a4b..0b25867a66 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -32,9 +32,10 @@ use futures::{Stream, StreamExt}; use hyper_util::rt::TokioIo; use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, - DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement as ProtoGatewayListenerRequirement, + CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, + DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, + DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement as ProtoGatewayListenerRequirement, GetCapabilitiesRequest, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, @@ -567,6 +568,22 @@ impl ComputeDriver for RemoteComputeDriver { let stream = response.into_inner(); Ok(tonic::Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.delete_workspace(request).await + } } #[derive(Clone)] @@ -870,6 +887,30 @@ impl ComputeRuntime { &self.gateway_listener_requirements } + pub(crate) async fn ensure_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + self.driver + .call("driver.ensure_workspace", None, |driver| async move { + driver + .ensure_workspace(Request::new(EnsureWorkspaceRequest { workspace })) + .await + }) + .await + .map(|_| ()) + } + + pub(crate) async fn delete_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + self.driver + .call("driver.delete_workspace", None, |driver| async move { + driver + .delete_workspace(Request::new(DeleteWorkspaceRequest { workspace })) + .await + }) + .await + .map(|_| ()) + } + pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; @@ -3881,6 +3922,20 @@ impl ComputeDriver for NoopTestDriver { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(futures::stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] @@ -4158,6 +4213,20 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } #[derive(Clone)] @@ -4484,6 +4553,20 @@ mod tests { UnboundedReceiverStream::new(receiver), ))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } async fn test_runtime(driver: SharedComputeDriver) -> ComputeRuntime { @@ -5250,6 +5333,20 @@ mod tests { ) -> Result, Status> { self.0.watch_sandboxes(request).await } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.delete_workspace(request).await + } } let runtime = test_runtime(Arc::new(FailingDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 6dc4318a8a..4d67eee018 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -2078,6 +2078,9 @@ pub(super) async fn handle_create_provider( metadata.workspace.clone_from(&workspace); } let provider_type = provider.r#type.clone(); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3233,6 +3236,9 @@ pub(super) async fn handle_update_provider( provider .credential_expires_at_ms .extend(req.credential_expires_at_ms); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3648,6 +3654,7 @@ pub(super) async fn handle_rotate_provider_credential( state.store.as_ref(), &workspace, Some(&state.credentials), + Some(&state.compute), provider_name, credential_key, ) diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index a22a195226..f2863511f1 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -435,6 +435,10 @@ pub(super) async fn handle_delete_workspace( } })?; + if deleted && let Err(e) = state.compute.delete_workspace(&name).await { + tracing::warn!(workspace = %name, error = %e, "failed to delete workspace platform resources"); + } + Ok(Response::new(DeleteWorkspaceResponse { deleted })) } diff --git a/crates/openshell-server/src/provider_refresh.rs b/crates/openshell-server/src/provider_refresh.rs index 9a655babb4..dc039265f6 100644 --- a/crates/openshell-server/src/provider_refresh.rs +++ b/crates/openshell-server/src/provider_refresh.rs @@ -343,6 +343,7 @@ pub async fn refresh_provider_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider_name: &str, credential_key: &str, ) -> Result { @@ -448,6 +449,7 @@ pub async fn refresh_provider_credential( store, workspace, credentials, + compute, &provider, credential_key, &minted, @@ -511,6 +513,7 @@ async fn apply_minted_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider: &Provider, credential_key: &str, minted: &MintedCredential, @@ -520,6 +523,9 @@ async fn apply_minted_credential( let staged_handles = if let Some(credentials) = credentials && credentials.stores_provider_credentials() { + if let Some(compute) = compute { + compute.ensure_workspace(workspace).await?; + } let mut creds_to_store = HashMap::from([(credential_key.to_string(), minted.access_token.clone())]); for (key, value) in &minted.additional_credentials { @@ -1115,8 +1121,12 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { ticker.tick().await; - if let Err(err) = - run_refresh_worker_tick(state.store.as_ref(), Some(&state.credentials)).await + if let Err(err) = run_refresh_worker_tick( + state.store.as_ref(), + Some(&state.credentials), + Some(&state.compute), + ) + .await { warn!(error = %err, "provider credential refresh worker tick failed"); } @@ -1136,6 +1146,7 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: async fn run_refresh_worker_tick( store: &Store, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, ) -> Result<(), Status> { let now_ms = current_time_ms(); let states = list_all_refresh_states(store).await.inspect_err(|_| { @@ -1200,6 +1211,7 @@ async fn run_refresh_worker_tick( store, state.object_workspace(), credentials, + compute, &state.provider_name, &state.credential_key, ) @@ -1328,6 +1340,7 @@ mod tests { &store, "default", None, + None, "my-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1398,6 +1411,7 @@ mod tests { &store, "default", Some(&credentials), + None, "my-stored-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1498,6 +1512,7 @@ mod tests { &store, "default", None, + None, "refreshing-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1578,6 +1593,7 @@ mod tests { &store, "default", None, + None, "my-delegated-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1672,6 +1688,7 @@ mod tests { &store, "default", None, + None, "my-drive", "GOOGLE_DRIVE_ACCESS_TOKEN", ) @@ -1715,7 +1732,7 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let stored_state = get_refresh_state( &store, @@ -1751,7 +1768,7 @@ mod tests { let store = test_store().await; let traced = test_exporter::install_traced(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let spans = traced.finished_spans(); let root = spans @@ -1873,6 +1890,7 @@ mod tests { &store, "default", None, + None, "aws-sts-test", "AWS_ACCESS_KEY_ID", ) @@ -1970,6 +1988,7 @@ mod tests { &store, "default", None, + None, "aws-sts-custom", "AWS_ACCESS_KEY_ID", ) @@ -2046,6 +2065,7 @@ mod tests { &store, "default", None, + None, "aws-sts-partial", "AWS_ACCESS_KEY_ID", ) @@ -2088,9 +2108,17 @@ mod tests { ]), }; - apply_minted_credential(&store, "default", None, &prov, "AWS_ACCESS_KEY_ID", &minted) - .await - .unwrap(); + apply_minted_credential( + &store, + "default", + None, + None, + &prov, + "AWS_ACCESS_KEY_ID", + &minted, + ) + .await + .unwrap(); let stored = store .get_message_by_name::("default", "aws-test") @@ -2163,6 +2191,7 @@ mod tests { &store, "default", Some(&credentials), + None, &prov, "AWS_ACCESS_KEY_ID", &minted, @@ -2266,6 +2295,7 @@ mod tests { &store, "default", Some(&credentials), + None, &refreshing_provider, "AWS_ACCESS_KEY_ID", &minted, @@ -2376,6 +2406,7 @@ mod tests { &store, "default", None, + None, "aws-sts-session", "AWS_ACCESS_KEY_ID", ) @@ -2444,6 +2475,7 @@ mod tests { &store, "default", None, + None, "aws-sts-lonesession", "AWS_ACCESS_KEY_ID", ) @@ -2526,8 +2558,14 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - let rotate = - refresh_provider_credential(&store, "default", None, "aws-race", "AWS_ACCESS_KEY_ID"); + let rotate = refresh_provider_credential( + &store, + "default", + None, + None, + "aws-race", + "AWS_ACCESS_KEY_ID", + ); let interfere = async { // Wait until the rotation is inside the STS call (its state read has // already happened), then delete the refresh and release STS. @@ -2645,6 +2683,7 @@ mod tests { &store, "default", None, + None, "aws-superseded", "AWS_ACCESS_KEY_ID", ); diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 016e8d56af..409d43180d 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -8,10 +8,12 @@ use futures::{Stream, stream}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverSandbox, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverSandbox, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, @@ -400,4 +402,18 @@ impl ComputeDriver for FakeComputeDriver { self.with_state(|state| state.calls.push(FakeComputeDriverCall::WatchSandboxes)); Ok(Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 66102ab751..c05667e29f 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -73,6 +73,28 @@ rules: verbs: - get {{- end }} + {{- if ne $workspaceMode "shared" }} + # TLS secret sync: read the source Secret in the release namespace and + # create/update copies in workspace namespaces so sandbox pods can mount + # client TLS material for mTLS to the gateway. + {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # Managed mode with kubernetes-secrets credential driver: credentials are + # stored as Secrets in workspace namespaces, requiring patch+delete in + # addition to the TLS sync verbs. + {{- end }} + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + - patch + - delete + {{- end }} + {{- end }} {{- if eq $workspaceMode "managed" }} # ServiceAccount creation in managed namespaces. - apiGroups: diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index f18827478b..1fa9bd1744 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -166,6 +166,33 @@ async fn managed_creates_namespace_with_labels() { let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); + + // Verify the sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox list should find mgd-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox get should resolve mgd-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane (uses sandbox_lookup_selector). + let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + // Verify sandbox is gone from the control plane after deletion. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list after delete failed: {out}"); + assert!( + !out.contains("mgd-sb"), + "sandbox list should NOT find mgd-sb after deletion: {out}" + ); } #[tokio::test] @@ -226,6 +253,18 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { out.contains("sb-b"), "sb-b CR should still be present: {out}" ); + + // Verify sb-b is still resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("sb-b"), + "sandbox list should find sb-b via control plane: {out}" + ); + assert!( + !out.contains("sb-a"), + "sandbox list should NOT find deleted sb-a: {out}" + ); } #[tokio::test] @@ -301,4 +340,27 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { !out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}" ); + + // Verify workspace isolation through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_a]).await; + assert!(ok, "sandbox list ws_a failed: {out}"); + assert!( + out.contains("sb-iso-a"), + "sandbox list ws_a should find sb-iso-a: {out}" + ); + assert!( + !out.contains("sb-iso-b"), + "sandbox list ws_a should NOT find sb-iso-b: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_b]).await; + assert!(ok, "sandbox list ws_b failed: {out}"); + assert!( + out.contains("sb-iso-b"), + "sandbox list ws_b should find sb-iso-b: {out}" + ); + assert!( + !out.contains("sb-iso-a"), + "sandbox list ws_b should NOT find sb-iso-a: {out}" + ); } diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index a382de446c..ddd94f8a1d 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -182,10 +182,33 @@ async fn operator_sandbox_in_labeled_namespace() { "sandbox CR name should be bare 'op-sb', got: {out}" ); - // Clean up. + // Verify sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox list should find op-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox get should resolve op-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane. let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; assert!(ok, "sandbox delete failed: {out}"); + // Verify sandbox is gone after deletion. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; + assert!(ok, "sandbox list after delete failed: {out}"); + assert!( + !out.contains("op-sb"), + "sandbox list should NOT find op-sb after deletion: {out}" + ); + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; assert!(ok, "workspace delete failed: {out}"); diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 3a0b7609ab..0ce4f61539 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -51,6 +51,13 @@ service ComputeDriver { // Stream sandbox observations from the platform. rpc WatchSandboxes(WatchSandboxesRequest) returns (stream WatchSandboxesEvent); + + // Ensure platform resources for a workspace exist (e.g. namespace). + // Idempotent: succeeds if resources already exist. + rpc EnsureWorkspace(EnsureWorkspaceRequest) returns (EnsureWorkspaceResponse); + + // Tear down platform resources for a workspace. + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); } message GetCapabilitiesRequest {} @@ -339,3 +346,17 @@ message WatchSandboxesEvent { WatchSandboxesPlatformEvent platform_event = 3; } } + +message EnsureWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message EnsureWorkspaceResponse {} + +message DeleteWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message DeleteWorkspaceResponse {} diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index f9ac186f52..82b8d5cfc8 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -230,11 +230,13 @@ preload_sandbox_image() { docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" fi + # Save without --platform: the platform-specific pull already constrained the + # local image, and --platform fails on OCI index (multi-arch) manifests. tmp="$(mktemp "${TMPDIR:-/tmp}/openshell-sandbox-image.XXXXXX")" - if ! docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then + if ! docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then echo "Pulling sandbox image for ${platform}..." docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" - docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" + docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" fi if ! k3d image import "${tmp}" --cluster "${CLUSTER_NAME}"; then From a73fe38badef8588439a525c8af0dddb044854eb Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Mon, 10 Aug 2026 21:49:33 -0400 Subject: [PATCH 06/16] fix(k8s): address re-review findings and add test coverage - Use server-side apply for TLS secret sync (fixes second sandbox creation failure when TLS is enabled) - Scope gateway-ID label selector unconditionally across all workspace modes (fixes operator reads/watches/deletes seeing foreign sandboxes) - Validate operator allowlist in EnsureWorkspace and DeleteWorkspace RPCs (prevents credential writes to namespaces outside the allowlist) - Extend ClusterRole secrets patch+delete to all non-shared modes with credential driver enabled (fixes operator credential storage RBAC) - Validate namespace ownership on 409 conflict in ensure_namespace (prevents adopting unowned namespaces in managed mode) - Replace delete_namespace_if_empty with unconditional delete_namespace letting Kubernetes cascade cleanup (fixes stuck terminating CRs) - Strengthen NetworkPolicy TODO to cover both managed and operator modes - Extract selector and ownership logic into testable free functions - Add unit tests for gateway-ID selectors and namespace ownership - Add Helm ClusterRole RBAC tests for operator credential driver Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 218 +++++++++++------- .../openshell-driver-kubernetes/src/grpc.rs | 44 +++- .../helm/openshell/templates/clusterrole.yaml | 9 +- .../openshell/tests/clusterrole_test.yaml | 44 ++++ 4 files changed, 214 insertions(+), 101 deletions(-) create mode 100644 deploy/helm/openshell/tests/clusterrole_test.yaml diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 7c6379a093..af766f0054 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -565,13 +565,15 @@ impl KubernetesComputeDriver { /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. /// - /// TODO: no `NetworkPolicy` is created here. The Helm-managed namespace gets - /// an SSH-isolation policy (port 2222 restricted to the gateway pod), but - /// managed-mode namespaces do not. Current risk is low: only sandbox pods - /// from the same workspace run in this namespace, so there is no lateral - /// movement target. A same-cluster `namespaceSelector` policy would also - /// break cross-cluster topologies where the gateway is external. Add a - /// configurable `NetworkPolicy` when mixed-workload or cross-cluster managed + /// TODO: no `NetworkPolicy` is created in dynamic namespaces. The + /// Helm-managed static namespace gets an SSH-isolation policy (port 2222 + /// restricted to the gateway pod), but managed and operator namespaces do + /// not. In managed mode, risk is low: only sandbox pods from the same + /// workspace run in the namespace, so there is no lateral movement target. + /// In operator mode, the admin owns the namespace and is responsible for + /// applying appropriate policies. A same-cluster `namespaceSelector` policy + /// would also break cross-cluster topologies where the gateway is external. + /// Add a configurable `NetworkPolicy` when mixed-workload or cross-cluster /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); @@ -627,6 +629,24 @@ impl KubernetesComputeDriver { info!(namespace = %ns_name, workspace = %workspace, "created managed namespace"); } Ok(Err(KubeError::Api(api))) if api.code == 409 => { + let existing = + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading namespace {ns_name}" + ))); + } + }; + if !is_namespace_owned_by_gateway( + existing.metadata.labels.as_ref(), + &self.config.gateway_id, + ) { + return Err(KubernetesDriverError::Precondition(format!( + "namespace {ns_name} exists but is not owned by this gateway" + ))); + } debug!(namespace = %ns_name, "managed namespace already exists"); } Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), @@ -726,7 +746,11 @@ impl KubernetesComputeDriver { match tokio::time::timeout( KUBE_API_TIMEOUT, - target_api.create(&PostParams::default(), ©), + target_api.patch( + &self.config.client_tls_secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), ) .await { @@ -734,39 +758,13 @@ impl KubernetesComputeDriver { info!( namespace = %namespace, secret = %self.config.client_tls_secret_name, - "created TLS secret copy" + "applied TLS secret copy" ); } - Ok(Err(KubeError::Api(api))) if api.code == 409 => { - match tokio::time::timeout( - KUBE_API_TIMEOUT, - target_api.replace( - &self.config.client_tls_secret_name, - &PostParams::default(), - ©, - ), - ) - .await - { - Ok(Ok(_)) => { - debug!( - namespace = %namespace, - secret = %self.config.client_tls_secret_name, - "updated TLS secret copy" - ); - } - Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout updating TLS secret in {namespace}" - ))); - } - } - } Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), Err(_) => { return Err(KubernetesDriverError::Message(format!( - "timeout creating TLS secret in {namespace}" + "timeout applying TLS secret in {namespace}" ))); } } @@ -774,35 +772,11 @@ impl KubernetesComputeDriver { Ok(()) } - /// Delete the managed namespace if it contains no sandboxes (managed mode - /// only). Called via the `DeleteWorkspace` RPC after workspace deletion. - pub async fn delete_namespace_if_empty( - &self, - workspace: &str, - ) -> Result<(), KubernetesDriverError> { + /// Delete the managed namespace and all its contents (managed mode only). + /// Called via the `DeleteWorkspace` RPC after workspace deletion. + /// Kubernetes cascades namespace deletion to all resources within it. + pub async fn delete_namespace(&self, workspace: &str) -> Result<(), KubernetesDriverError> { let ns_name = managed_namespace(&self.config.gateway_id, workspace); - - let sandbox_api_version = self - .supported_sandbox_api_version(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let agent_api = Self::agent_sandbox_api(self.client.clone(), sandbox_api_version, &ns_name); - - let lp = ListParams::default() - .labels(&openshell_sandbox_label_selector()) - .limit(1); - let list = tokio::time::timeout(KUBE_API_TIMEOUT, agent_api.api.list(&lp)) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!("timeout listing sandboxes in {ns_name}")) - })? - .map_err(KubernetesDriverError::from_kube)?; - - if !list.items.is_empty() { - debug!(namespace = %ns_name, "namespace still has sandboxes, skipping delete"); - return Ok(()); - } - let ns_api: Api = Api::all(self.client.clone()); let ns = match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { @@ -819,14 +793,7 @@ impl KubernetesComputeDriver { } }; - let labels = ns.metadata.labels.as_ref(); - let is_owned = labels - .and_then(|l| l.get(LABEL_MANAGED_BY)) - .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) - && labels - .and_then(|l| l.get(LABEL_GATEWAY_ID)) - .is_some_and(|v| v == &self.config.gateway_id); - if !is_owned { + if !is_namespace_owned_by_gateway(ns.metadata.labels.as_ref(), &self.config.gateway_id) { debug!( namespace = %ns_name, "namespace not owned by this gateway, skipping delete" @@ -841,7 +808,7 @@ impl KubernetesComputeDriver { .await { Ok(Ok(_)) => { - info!(namespace = %ns_name, workspace = %workspace, "deleted empty managed namespace"); + info!(namespace = %ns_name, workspace = %workspace, "deleted managed namespace"); } Ok(Err(KubeError::Api(api))) if api.code == 404 => { debug!(namespace = %ns_name, "managed namespace already deleted"); @@ -919,22 +886,11 @@ impl KubernetesComputeDriver { } fn sandbox_lookup_selector(&self, sandbox_id: &str) -> String { - let mut selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); - if self.config.workspace_mode == WorkspaceMode::Managed { - use std::fmt::Write; - write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); - } - selector + sandbox_lookup_selector_for(sandbox_id, &self.config.gateway_id) } fn openshell_sandbox_selector(&self) -> String { - let mut selector = openshell_sandbox_label_selector(); - if self.config.workspace_mode == WorkspaceMode::Managed { - use std::fmt::Write; - write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); - } - selector + openshell_sandbox_selector_for(&self.config.gateway_id) } async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { @@ -1844,6 +1800,31 @@ fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), Ok(()) } +fn is_namespace_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id) +} + +fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { + format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" + ) +} + +fn openshell_sandbox_selector_for(gateway_id: &str) -> String { + use std::fmt::Write; + let mut selector = openshell_sandbox_label_selector(); + write!(selector, ",{LABEL_GATEWAY_ID}={gateway_id}").unwrap(); + selector +} + fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -7378,4 +7359,69 @@ mod tests { ); assert_eq!(volume["secret"]["defaultMode"], 0o440); } + + #[test] + fn sandbox_lookup_selector_always_includes_gateway_id() { + let sel = sandbox_lookup_selector_for("sb-123", "gw-42"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-42")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_SANDBOX_ID}=sb-123")), + "selector must include sandbox ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn openshell_sandbox_selector_always_includes_gateway_id() { + let sel = openshell_sandbox_selector_for("gw-99"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-99")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn namespace_owned_with_correct_labels() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-1".to_string()), + ]); + assert!(is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_missing_managed_by() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_wrong_gateway_id() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-other".to_string()), + ]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_no_labels() { + assert!(!is_namespace_owned_by_gateway(None, "gw-1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index fbfcecaac1..cb8817c930 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -184,11 +184,23 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } - if self.driver.workspace_mode() == WorkspaceMode::Managed { - self.driver - .ensure_namespace(&workspace) - .await - .map_err(|e| Status::internal(e.to_string()))?; + match self.driver.workspace_mode() { + WorkspaceMode::Managed => { + self.driver + .ensure_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + WorkspaceMode::Operator => { + if let Some(allowlist) = self.driver.operator_allowlist() + && !allowlist.contains(&workspace) + { + return Err(Status::permission_denied(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + } + WorkspaceMode::Shared => {} } Ok(Response::new(EnsureWorkspaceResponse {})) } @@ -201,11 +213,23 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } - if self.driver.workspace_mode() == WorkspaceMode::Managed { - self.driver - .delete_namespace_if_empty(&workspace) - .await - .map_err(|e| Status::internal(e.to_string()))?; + match self.driver.workspace_mode() { + WorkspaceMode::Managed => { + self.driver + .delete_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + WorkspaceMode::Operator => { + if let Some(allowlist) = self.driver.operator_allowlist() + && !allowlist.contains(&workspace) + { + return Err(Status::permission_denied(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + } + WorkspaceMode::Shared => {} } Ok(Response::new(DeleteWorkspaceResponse {})) } diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index c05667e29f..299ee9ca3d 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -77,10 +77,9 @@ rules: # TLS secret sync: read the source Secret in the release namespace and # create/update copies in workspace namespaces so sandbox pods can mount # client TLS material for mTLS to the gateway. - {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - # Managed mode with kubernetes-secrets credential driver: credentials are - # stored as Secrets in workspace namespaces, requiring patch+delete in - # addition to the TLS sync verbs. + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # kubernetes-secrets credential driver: credentials are stored as Secrets + # in workspace namespaces, requiring patch+delete in addition to TLS sync. {{- end }} - apiGroups: - "" @@ -90,7 +89,7 @@ rules: - get - create - update - {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - patch - delete {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml new file mode 100644 index 0000000000..c2363b08eb --- /dev/null +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: ClusterRole RBAC +templates: + - templates/clusterrole.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: grants secrets patch and delete when credential driver is enabled (operator) + set: + server.drivers.kubernetes.workspaceMode: operator + server.credentialDrivers.kubernetesSecrets.enabled: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch", "delete"] + + - it: omits secrets patch and delete when credential driver is disabled (operator) + set: + server.drivers.kubernetes.workspaceMode: operator + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + + - it: omits secrets rule entirely in shared mode + set: + server.drivers.kubernetes.workspaceMode: shared + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + any: true From dfdb5b10845cdef28fe5f2e2ce6d2c26dd2bcc4b Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Tue, 11 Aug 2026 09:54:17 -0400 Subject: [PATCH 07/16] ci(k8s): add workspace managed and operator mode e2e to CI Wire the existing e2e:kubernetes:workspace-managed and e2e:kubernetes:workspace-operator mise tasks into the branch-e2e workflow so they run alongside the other core Kubernetes e2e suites. Both are gated by run_core_e2e and included in the Core E2E result gate. Signed-off-by: Derek Carr --- .github/workflows/branch-e2e.yml | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3d68746b82..37154c75df 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -182,6 +182,34 @@ jobs: extra-helm-values: ${{ matrix.extra_helm_values }} cli-artifact-prefix: rust-binary-cli + kubernetes-workspace-managed-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace managed mode) + e2e-task: e2e:kubernetes:workspace-managed + cli-artifact-prefix: rust-binary-cli + + kubernetes-workspace-operator-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace operator mode) + e2e-task: e2e:kubernetes:workspace-operator + cli-artifact-prefix: rust-binary-cli + kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' @@ -212,7 +240,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -224,6 +252,8 @@ jobs: BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} + KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} + KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} run: | set -euo pipefail failed=0 @@ -233,7 +263,9 @@ jobs: "build-cli:$BUILD_CLI_RESULT" \ "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ - "kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do + "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ + "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ + "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" if [ "$result" != "success" ]; then From 597b2ee1db3645205094393c69ebf98296d54a93 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Tue, 11 Aug 2026 12:56:54 -0400 Subject: [PATCH 08/16] test(k8s): add e2e tests for workspace namespace modes Add 7 new e2e tests covering workspace namespace lifecycle, TLS secret copying, ownership conflict detection, DNS-1123 validation, operator namespace preservation, and dynamic label watcher behavior. Fix async sandbox deletion race condition in existing tests by polling sandbox list instead of asserting immediately after delete. Signed-off-by: Derek Carr --- e2e/rust/tests/workspace_namespace_managed.rs | 355 +++++++++++++++++- .../tests/workspace_namespace_operator.rs | 169 ++++++++- 2 files changed, 504 insertions(+), 20 deletions(-) diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 1fa9bd1744..6bc0564ccf 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -70,6 +70,22 @@ fn unique_workspace(prefix: &str) -> String { format!("{prefix}-{ts}") } +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + struct ManagedCleanup { workspace: String, sandboxes: Vec, @@ -186,13 +202,8 @@ async fn managed_creates_namespace_with_labels() { let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; assert!(ok, "sandbox delete failed: {out}"); - // Verify sandbox is gone from the control plane after deletion. - let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; - assert!(ok, "sandbox list after delete failed: {out}"); - assert!( - !out.contains("mgd-sb"), - "sandbox list should NOT find mgd-sb after deletion: {out}" - ); + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ws, "mgd-sb").await; } #[tokio::test] @@ -240,8 +251,8 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; assert!(ok, "sandbox sb-a delete failed: {out}"); - // Brief wait, then verify the namespace still exists. - tokio::time::sleep(Duration::from_secs(3)).await; + // Wait for sb-a to be fully removed before checking namespace state. + wait_sandbox_gone(&ws, "sb-a").await; let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!(ok, "managed namespace {ns} should still exist with sb-b"); @@ -261,10 +272,6 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { out.contains("sb-b"), "sandbox list should find sb-b via control plane: {out}" ); - assert!( - !out.contains("sb-a"), - "sandbox list should NOT find deleted sb-a: {out}" - ); } #[tokio::test] @@ -364,3 +371,325 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { "sandbox list ws_b should NOT find sb-iso-a: {out}" ); } + +#[tokio::test] +async fn managed_workspace_delete_removes_namespace() { + let ws = unique_workspace("mgddel"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["del-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "del-sb", + "--", + "echo", + "del-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("del-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should exist after sandbox create" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "del-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ws, "del-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after workspace delete"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_tls_secret_copied_to_namespace() { + let (ok, config_out) = kubectl(&[ + "get", + "configmap", + "openshell-config", + "-n", + "openshell", + "-o", + "jsonpath={.data.gateway\\.toml}", + ]) + .await; + if !ok || !config_out.contains("client_tls_secret_name") { + eprintln!("SKIP: client_tls_secret_name not configured; TLS secret copying disabled"); + return; + } + + let ws = unique_workspace("mgdtls"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["tls-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "tls-sb", + "--", + "echo", + "tls-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("tls-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, out) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; + assert!( + ok, + "TLS secret openshell-client-tls should be copied to managed namespace {ns}: {out}" + ); + + let (ok, label_out) = kubectl(&[ + "get", + "secret", + "openshell-client-tls", + "-n", + &ns, + "-o", + "jsonpath={.metadata.labels}", + ]) + .await; + assert!(ok, "failed to read TLS secret labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "copied TLS secret missing managed-by label: {label_out}" + ); +} + +#[tokio::test] +async fn managed_rejects_namespace_owned_by_different_gateway() { + let ws = unique_workspace("mgdown"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to pre-create namespace {ns}: {out}"); + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/managed-by=openshell", + "openshell.ai/gateway-id=wrong-gateway", + ]) + .await; + assert!(ok, "failed to label namespace: {out}"); + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "conflict-sb", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for namespace owned by different gateway, but succeeded: {out}" + ); +} + +#[tokio::test] +async fn managed_full_lifecycle_with_multiple_sandboxes() { + let ws = unique_workspace("mgdlc"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["lc-a".into(), "lc-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-a", + "--", + "echo", + "a", + ]) + .await; + assert!(ok, "sandbox lc-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-b", + "--", + "echo", + "b", + ]) + .await; + assert!(ok, "sandbox lc-b create failed: {out}"); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist"); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-a", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-a delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-a").await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should still exist with lc-b remaining" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-b", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-b delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-b").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after full lifecycle cleanup"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_rejects_invalid_dns1123_sandbox_name() { + let ws = unique_workspace("mgddns"); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "my_bad_name", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with underscore name should be rejected: {out}" + ); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "MyBadName", + "--", + "echo", + "nope", + ]) + .await; + assert!(!ok, "sandbox with uppercase name should be rejected: {out}"); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "trailing-", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with trailing hyphen should be rejected: {out}" + ); + let normalized: String = out + .chars() + .filter(|c| *c != '│') + .collect::() + .split_whitespace() + .collect::>() + .join(" "); + assert!( + normalized.contains("must not start or end with a hyphen"), + "error should mention hyphen constraint: {out}" + ); +} diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index ddd94f8a1d..f421aa6ada 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -65,6 +65,22 @@ fn unique_namespace(prefix: &str) -> String { format!("{prefix}-{ts}") } +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + async fn provision_operator_namespace(name: &str) { let (ok, out) = kubectl(&["create", "namespace", name]).await; assert!(ok, "failed to create namespace {name}: {out}"); @@ -201,13 +217,8 @@ async fn operator_sandbox_in_labeled_namespace() { let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; assert!(ok, "sandbox delete failed: {out}"); - // Verify sandbox is gone after deletion. - let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; - assert!(ok, "sandbox list after delete failed: {out}"); - assert!( - !out.contains("op-sb"), - "sandbox list should NOT find op-sb after deletion: {out}" - ); + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ns, "op-sb").await; let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; assert!(ok, "workspace delete failed: {out}"); @@ -293,3 +304,147 @@ async fn operator_rejects_nonexistent_namespace() { // Clean up. let _ = run_cli(&["workspace", "delete", &ns]).await; } + +#[tokio::test] +async fn operator_workspace_delete_preserves_namespace() { + let ns = unique_namespace("opdel"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["opdel-sb".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "opdel-sb", + "--", + "echo", + "opdel-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "opdel-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ns, "opdel-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "operator namespace {ns} should still exist after workspace delete: {out}" + ); + + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/e2e-operator-workspace"), + "operator label should be intact after workspace delete: {label_out}" + ); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_label_removal_blocks_sandbox_creation() { + let ns = unique_namespace("oplbl"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["lbl-sb1".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb1", + "--", + "echo", + "lbl-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "lbl-sb1", "--workspace", &ns]).await; + assert!(ok, "sandbox lbl-sb1 delete failed: {out}"); + + wait_sandbox_gone(&ns, "lbl-sb1").await; + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/e2e-operator-workspace-", + ]) + .await; + assert!(ok, "failed to remove operator label: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, _out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb2", + "--", + "echo", + "should-fail", + ]) + .await; + if !ok { + break; + } + // Sandbox was created despite label removal — clean it up and retry. + let _ = run_cli(&["sandbox", "delete", "lbl-sb2", "--workspace", &ns]).await; + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox creation still succeeds 30s after operator label removal; \ + watcher did not remove namespace from allowlist" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + delete_namespace(&ns).await; +} From e625ca8abdf162327d44c8cb8c15254d51d01f72 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Wed, 12 Aug 2026 11:33:38 -0400 Subject: [PATCH 09/16] fix(k8s): grant secrets/patch unconditionally and backfill gateway-id labels Address two review findings: 1. RBAC: server-side apply (PATCH) is used for TLS secret sync in multi-namespace modes, but the ClusterRole only granted patch when the kubernetes-secrets credential driver was enabled. Grant patch unconditionally for non-shared modes since TLS sync always needs it; keep delete gated on the credential driver. 2. Upgrade safety: the new gateway-id label selector would orphan legacy Sandbox CRs that predate its introduction. Add a startup backfill in shared mode that patches any managed Sandbox CR missing the gateway-id label before the driver begins serving requests. Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 72 +++++++++++++++++++ crates/openshell-server/src/compute/mod.rs | 5 +- .../helm/openshell/templates/clusterrole.yaml | 14 ++-- .../openshell/tests/clusterrole_test.yaml | 4 +- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index af766f0054..f3f5142043 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -559,6 +559,78 @@ impl KubernetesComputeDriver { self.config.workspace_mode } + /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that + /// predate its introduction. Runs once at startup in shared mode so that + /// label-selector based lookups continue to find legacy resources. + pub async fn backfill_gateway_id_labels(&self) { + let sandbox_api = match self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + { + Ok(api) => api, + Err(e) => { + warn!(error = %e, "skipping gateway-id label backfill: cannot resolve Sandbox API"); + return; + } + }; + + let selector = openshell_sandbox_label_selector(); + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .list(&ListParams::default().labels(&selector)), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(e)) => { + warn!(error = %e, "skipping gateway-id label backfill: list failed"); + return; + } + Err(_) => { + warn!("skipping gateway-id label backfill: list timed out"); + return; + } + }; + + let gateway_id = &self.config.gateway_id; + for obj in &list { + let has_label = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id); + if has_label { + continue; + } + let name = match obj.metadata.name.as_deref() { + Some(n) => n, + None => continue, + }; + let patch = serde_json::json!({ + "metadata": { + "labels": { + LABEL_GATEWAY_ID: gateway_id + } + } + }); + match sandbox_api + .api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + { + Ok(_) => { + info!(sandbox = %name, gateway_id, "backfilled gateway-id label"); + } + Err(e) => { + warn!(sandbox = %name, error = %e, "failed to backfill gateway-id label"); + } + } + } + } + /// Ensure the K8s namespace for a workspace exists (managed mode only). /// /// Idempotent: returns the namespace name whether it was just created or diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0b25867a66..c21ef8446d 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -54,7 +54,7 @@ use openshell_driver_docker::DockerComputeDriver; #[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, - OperatorNamespaceAllowlist, + OperatorNamespaceAllowlist, WorkspaceMode, }; #[cfg(not(target_os = "windows"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; @@ -797,6 +797,9 @@ impl ComputeRuntime { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + if driver.workspace_mode() == WorkspaceMode::Shared { + driver.backfill_gateway_id_labels().await; + } let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); let runtime = Self::from_driver( diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 299ee9ca3d..0f62d8e915 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -74,13 +74,11 @@ rules: - get {{- end }} {{- if ne $workspaceMode "shared" }} - # TLS secret sync: read the source Secret in the release namespace and - # create/update copies in workspace namespaces so sandbox pods can mount - # client TLS material for mTLS to the gateway. - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - # kubernetes-secrets credential driver: credentials are stored as Secrets - # in workspace namespaces, requiring patch+delete in addition to TLS sync. - {{- end }} + # Secrets access for multi-namespace modes: + # - TLS sync uses server-side apply (patch) to copy the client TLS Secret + # into workspace namespaces so sandbox pods can mount mTLS material. + # - The kubernetes-secrets credential driver stores credentials as Secrets + # in workspace namespaces, additionally requiring delete for cleanup. - apiGroups: - "" resources: @@ -89,8 +87,8 @@ rules: - get - create - update - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - patch + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - delete {{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index c2363b08eb..efaa117f44 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -21,7 +21,7 @@ tests: resources: ["secrets"] verbs: ["get", "create", "update", "patch", "delete"] - - it: omits secrets patch and delete when credential driver is disabled (operator) + - it: grants secrets patch for TLS sync even when credential driver is disabled (operator) set: server.drivers.kubernetes.workspaceMode: operator asserts: @@ -30,7 +30,7 @@ tests: content: apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "update"] + verbs: ["get", "create", "update", "patch"] - it: omits secrets rule entirely in shared mode set: From 050941885f75534bd0a76aa2a780b1ba97a002ff Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Wed, 12 Aug 2026 12:41:34 -0400 Subject: [PATCH 10/16] fix(k8s): address workspace namespace review findings Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 18 +- .../openshell-driver-kubernetes/src/config.rs | 63 ++- .../openshell-driver-kubernetes/src/driver.rs | 390 +++++++++++++++--- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 35 +- crates/openshell-server/src/compute/mod.rs | 40 +- crates/openshell-server/src/grpc/mod.rs | 31 +- crates/openshell-server/src/grpc/workspace.rs | 73 +++- deploy/helm/openshell/README.md | 2 +- .../ci/values-workspace-managed.yaml | 2 + .../helm/openshell/templates/clusterrole.yaml | 12 + .../openshell/templates/gateway-config.yaml | 5 + .../openshell/tests/clusterrole_test.yaml | 23 ++ .../openshell/tests/gateway_config_test.yaml | 7 + deploy/helm/openshell/values.yaml | 3 +- docs/reference/gateway-config.mdx | 11 + docs/reference/sandbox-compute-drivers.mdx | 3 +- e2e/rust/tests/workspace_namespace_managed.rs | 56 ++- e2e/with-kube-gateway.sh | 8 + tasks/test.toml | 2 +- 20 files changed, 715 insertions(+), 73 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index b5994eacdd..0786b0bcad 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -327,14 +327,24 @@ embed the workspace prefix for collision avoidance. No namespace lifecycle management. RBAC uses a namespace-scoped Role. **Managed** auto-creates a K8s namespace per workspace on first sandbox create. -Each new namespace receives a ServiceAccount and copies OpenShift SCC UID-range -and supplemental-group annotations from the gateway namespace when present. The -driver deletes the namespace when the last sandbox in it is removed -(`delete_namespace_if_empty`). Requires a non-empty `gateway_id` (validated as a +Each new namespace receives a ServiceAccount and the configured gateway-only +SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the +driver's source namespace on every sandbox create so registry credential +rotations propagate. The namespace also copies OpenShift SCC UID-range and +supplemental-group annotations from the gateway namespace when present. The +driver deletes the namespace during workspace deletion. The workspace remains +durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a +transient failure can be retried. Namespace deletion uses the fetched UID as a +precondition to avoid deleting a replacement namespace. Requires a non-empty +`gateway_id` (validated as a DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace `create`/`delete` and ServiceAccount `create`/`get` permissions. +Operator mode does not create NetworkPolicies or copy image-pull Secrets. +Platform teams must apply the gateway ingress boundary and provision configured +image-pull Secrets in every operator-managed namespace. + **Operator** uses pre-provisioned namespaces discovered through two optional sources: a K8s label selector (`operator_namespace_label`) and a drop-in allowlist file (`operator_namespace_file`). At least one must be configured. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 2df422e588..04955acb70 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -3,7 +3,7 @@ use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, RwLock}; @@ -304,6 +304,10 @@ pub struct KubernetesComputeConfig { pub image_pull_policy: String, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, + /// Managed-mode SSH ingress isolation. When enabled, the driver creates a + /// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222 + /// only from gateway pods matching this peer. + pub managed_ssh_ingress: ManagedSshIngressConfig, /// Image that provides the `openshell-sandbox` supervisor binary. /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. @@ -392,6 +396,14 @@ pub struct KubernetesComputeConfig { pub sandbox_gid: Option, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ManagedSshIngressConfig { + pub enabled: bool, + pub gateway_namespace: String, + pub gateway_pod_selector: BTreeMap, +} + /// Lower bound enforced by kubelet for projected SA tokens. pub const MIN_SA_TOKEN_TTL_SECS: i64 = 600; @@ -428,6 +440,7 @@ impl Default for KubernetesComputeConfig { // is Podman vocabulary and is not a valid Kubernetes value. image_pull_policy: String::new(), image_pull_secrets: Vec::new(), + managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), @@ -593,7 +606,7 @@ impl KubernetesComputeConfig { /// 3. Fallback defaults: UID=`1000`, GID=UID pub fn resolve_sandbox_uid( &self, - namespace_annotations: Option<&std::collections::BTreeMap>, + namespace_annotations: Option<&BTreeMap>, ) -> u32 { if let Some(uid) = self.sandbox_uid { return uid; @@ -611,7 +624,7 @@ impl KubernetesComputeConfig { pub fn resolve_sandbox_gid( &self, resolved_uid: u32, - _namespace_annotations: Option<&std::collections::BTreeMap>, + _namespace_annotations: Option<&BTreeMap>, ) -> u32 { self.sandbox_gid .or(self.sandbox_uid) @@ -738,6 +751,18 @@ impl KubernetesComputeConfig { prefix.len() )); } + if self.managed_ssh_ingress.enabled { + if self.managed_ssh_ingress.gateway_namespace.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_namespace".into() + ); + } + if self.managed_ssh_ingress.gateway_pod_selector.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_pod_selector".into(), + ); + } + } Ok(()) } WorkspaceMode::Operator => { @@ -1744,6 +1769,38 @@ mod tests { cfg.validate_workspace_mode().unwrap(); } + #[test] + fn validate_workspace_mode_managed_requires_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::new(), + }, + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("gateway_pod_selector"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + #[test] fn validate_workspace_mode_operator_requires_discovery() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f3f5142043..f9f0790e01 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -14,6 +14,12 @@ use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, ServiceAccount, Volume, VolumeMount, }; +use k8s_openapi::api::networking::v1::{ + NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, + NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use kube::api::{ Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, }; @@ -54,6 +60,8 @@ use tracing::{debug, info, warn}; pub type WatchStream = Pin> + Send>>; +const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; + #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { #[error("sandbox already exists")] @@ -562,17 +570,11 @@ impl KubernetesComputeDriver { /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that /// predate its introduction. Runs once at startup in shared mode so that /// label-selector based lookups continue to find legacy resources. - pub async fn backfill_gateway_id_labels(&self) { - let sandbox_api = match self + pub async fn backfill_gateway_id_labels(&self) -> Result<(), KubernetesDriverError> { + let sandbox_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await - { - Ok(api) => api, - Err(e) => { - warn!(error = %e, "skipping gateway-id label backfill: cannot resolve Sandbox API"); - return; - } - }; + .map_err(KubernetesDriverError::Message)?; let selector = openshell_sandbox_label_selector(); let list = match tokio::time::timeout( @@ -584,30 +586,21 @@ impl KubernetesComputeDriver { .await { Ok(Ok(list)) => list, - Ok(Err(e)) => { - warn!(error = %e, "skipping gateway-id label backfill: list failed"); - return; - } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), Err(_) => { - warn!("skipping gateway-id label backfill: list timed out"); - return; + return Err(KubernetesDriverError::Message( + "timeout listing Sandbox resources for gateway-id label backfill".to_string(), + )); } }; let gateway_id = &self.config.gateway_id; for obj in &list { - let has_label = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_GATEWAY_ID)) - .is_some_and(|v| v == gateway_id); - if has_label { + if !gateway_id_label_needs_backfill(obj.metadata.labels.as_ref(), gateway_id) { continue; } - let name = match obj.metadata.name.as_deref() { - Some(n) => n, - None => continue, + let Some(name) = obj.metadata.name.as_deref() else { + continue; }; let patch = serde_json::json!({ "metadata": { @@ -616,19 +609,27 @@ impl KubernetesComputeDriver { } } }); - match sandbox_api - .api - .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) - .await + match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await { - Ok(_) => { + Ok(Ok(_)) => { info!(sandbox = %name, gateway_id, "backfilled gateway-id label"); } - Err(e) => { - warn!(sandbox = %name, error = %e, "failed to backfill gateway-id label"); + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout backfilling gateway-id label on Sandbox {name}" + ))); } } } + + Ok(()) } /// Ensure the K8s namespace for a workspace exists (managed mode only). @@ -637,16 +638,6 @@ impl KubernetesComputeDriver { /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. /// - /// TODO: no `NetworkPolicy` is created in dynamic namespaces. The - /// Helm-managed static namespace gets an SSH-isolation policy (port 2222 - /// restricted to the gateway pod), but managed and operator namespaces do - /// not. In managed mode, risk is low: only sandbox pods from the same - /// workspace run in the namespace, so there is no lateral movement target. - /// In operator mode, the admin owns the namespace and is responsible for - /// applying appropriate policies. A same-cluster `namespaceSelector` policy - /// would also break cross-cluster topologies where the gateway is external. - /// Add a configurable `NetworkPolicy` when mixed-workload or cross-cluster - /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); let ns_api: Api = Api::all(self.client.clone()); @@ -730,10 +721,42 @@ impl KubernetesComputeDriver { } self.ensure_service_account(&ns_name).await?; + self.ensure_managed_ssh_network_policy(&ns_name).await?; Ok(ns_name) } + async fn ensure_managed_ssh_network_policy( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + if !self.config.managed_ssh_ingress.enabled { + return Ok(()); + } + + let policy = managed_ssh_network_policy(namespace, &self.config); + let policy_api: Api = Api::namespaced(self.client.clone(), namespace); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policy_api.patch( + MANAGED_SSH_NETWORK_POLICY_NAME, + &PatchParams::apply("openshell"), + &Patch::Apply(&policy), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, "applied managed sandbox SSH NetworkPolicy"); + Ok(()) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message(format!( + "timeout applying SSH NetworkPolicy in {namespace}" + ))), + } + } + async fn ensure_service_account(&self, namespace: &str) -> Result<(), KubernetesDriverError> { let sa_api: Api = Api::namespaced(self.client.clone(), namespace); let sa = ServiceAccount { @@ -844,6 +867,62 @@ impl KubernetesComputeDriver { Ok(()) } + /// Copy the explicitly configured image-pull Secrets into a managed + /// workspace namespace. Server-side apply refreshes rotated credentials + /// without forcibly taking fields owned by another manager. + async fn ensure_image_pull_secrets( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + + for secret_name in &self.config.image_pull_secrets { + let source = match tokio::time::timeout(KUBE_API_TIMEOUT, source_api.get(secret_name)) + .await + { + Ok(Ok(secret)) => secret, + Ok(Err(KubeError::Api(error))) if error.code == 404 => { + return Err(KubernetesDriverError::Precondition(format!( + "configured image-pull Secret {secret_name} does not exist in source namespace {}", + self.config.namespace + ))); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading image-pull Secret {secret_name} from {}", + self.config.namespace + ))); + } + }; + + let copy = image_pull_secret_copy(secret_name, namespace, source); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.patch( + secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, secret = %secret_name, "applied image-pull Secret copy"); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout applying image-pull Secret {secret_name} in {namespace}" + ))); + } + } + } + + Ok(()) + } + /// Delete the managed namespace and all its contents (managed mode only). /// Called via the `DeleteWorkspace` RPC after workspace deletion. /// Kubernetes cascades namespace deletion to all resources within it. @@ -873,11 +952,14 @@ impl KubernetesComputeDriver { return Ok(()); } - match tokio::time::timeout( - KUBE_API_TIMEOUT, - ns_api.delete(&ns_name, &DeleteParams::default()), - ) - .await + let namespace_uid = ns.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message(format!( + "namespace {ns_name} has no UID; refusing an unguarded delete" + )) + })?; + let delete_params = namespace_delete_params(namespace_uid); + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.delete(&ns_name, &delete_params)).await { Ok(Ok(_)) => { info!(namespace = %ns_name, workspace = %workspace, "deleted managed namespace"); @@ -1254,7 +1336,11 @@ impl KubernetesComputeDriver { let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), - WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, + WorkspaceMode::Managed => { + let namespace = self.ensure_namespace(workspace).await?; + self.ensure_image_pull_secrets(&namespace).await?; + namespace + } WorkspaceMode::Operator => { if let Some(ref allowlist) = self.operator_allowlist && !allowlist.contains(workspace) @@ -1884,6 +1970,22 @@ fn is_namespace_owned_by_gateway( .is_some_and(|v| v == gateway_id) } +fn gateway_id_label_needs_backfill( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) + .is_none_or(|value| value != gateway_id) +} + +fn namespace_delete_params(uid: String) -> DeleteParams { + DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: None, + }) +} + fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { format!( "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" @@ -1915,6 +2017,70 @@ fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(MANAGED_SSH_NETWORK_POLICY_NAME.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + config.managed_ssh_ingress.gateway_namespace.clone(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(config.managed_ssh_ingress.gateway_pod_selector.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(2222)), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + status: None, + } +} + +fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + } +} + fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -7462,6 +7628,123 @@ mod tests { ); } + #[test] + fn gateway_id_backfill_adopts_unlabelled_sandbox() { + let labels = BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_adopts_sandbox_from_previous_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-old".to_string())]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_skips_sandbox_already_owned_by_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn managed_ssh_policy_allows_only_gateway_peer_on_port_2222() { + let config = KubernetesComputeConfig { + managed_ssh_ingress: crate::config::ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway-ns".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + let policy = managed_ssh_network_policy("workspace-ns", &config); + let spec = policy.spec.unwrap(); + assert_eq!( + spec.policy_types.as_deref(), + Some(["Ingress".to_string()].as_slice()) + ); + let ingress = &spec.ingress.unwrap()[0]; + assert_eq!( + ingress.ports.as_ref().unwrap()[0].port, + Some(IntOrString::Int(2222)) + ); + let peer = &ingress.from.as_ref().unwrap()[0]; + assert_eq!( + peer.namespace_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("kubernetes.io/metadata.name") + .map(String::as_str), + Some("gateway-ns") + ); + assert_eq!( + peer.pod_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("app.kubernetes.io/name") + .map(String::as_str), + Some("openshell") + ); + } + + #[test] + fn image_pull_secret_copy_keeps_only_portable_secret_fields() { + let source: Secret = serde_json::from_value(serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "gateway", + "uid": "source-uid", + "resourceVersion": "42", + "labels": { "source-only": "true" }, + "annotations": { "source-only": "true" }, + "finalizers": ["example.test/finalizer"] + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + })) + .unwrap(); + + let copy = image_pull_secret_copy("regcred", "workspace", source); + assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); + assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); + assert_eq!( + copy.type_.as_deref(), + Some("kubernetes.io/dockerconfigjson") + ); + assert!( + copy.data + .as_ref() + .unwrap() + .contains_key(".dockerconfigjson") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_MANAGED_BY) + .map(String::as_str), + Some(LABEL_MANAGED_BY_VALUE) + ); + assert!(copy.metadata.uid.is_none()); + assert!(copy.metadata.resource_version.is_none()); + assert!(copy.metadata.annotations.is_none()); + assert!(copy.metadata.finalizers.is_none()); + } + #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ @@ -7496,4 +7779,15 @@ mod tests { fn namespace_not_owned_no_labels() { assert!(!is_namespace_owned_by_gateway(None, "gw-1")); } + + #[test] + fn namespace_delete_is_guarded_by_fetched_uid() { + let params = namespace_delete_params("namespace-uid".to_string()); + assert_eq!( + params + .preconditions + .and_then(|preconditions| preconditions.uid), + Some("namespace-uid".to_string()) + ); + } } diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index d18a23a618..1a234385c6 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - OperatorNamespaceAllowlist, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + ManagedSshIngressConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 15acdec59a..75d56245c7 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -3,6 +3,7 @@ use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; +use std::collections::BTreeMap; use std::net::SocketAddr; use tracing::info; use tracing_subscriber::EnvFilter; @@ -12,7 +13,8 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, + WorkspaceMode, }; #[derive(Parser, Debug)] @@ -69,6 +71,19 @@ struct Args { )] sandbox_image_pull_secrets: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] + managed_ssh_ingress_enabled: bool, + + #[arg(long, env = "OPENSHELL_MANAGED_SSH_GATEWAY_NAMESPACE")] + managed_ssh_gateway_namespace: Option, + + #[arg( + long, + env = "OPENSHELL_MANAGED_SSH_GATEWAY_POD_SELECTOR", + value_delimiter = ',' + )] + managed_ssh_gateway_pod_selector: Vec, + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, @@ -173,6 +188,19 @@ async fn main() -> Result<()> { ) .init(); + let managed_ssh_gateway_pod_selector = args + .managed_ssh_gateway_pod_selector + .iter() + .map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.to_string(), value.to_string())) + .ok_or_else(|| { + miette::miette!("managed SSH gateway pod selector must use key=value: {entry}") + }) + }) + .collect::>>()?; + let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, @@ -183,6 +211,11 @@ async fn main() -> Result<()> { default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), image_pull_secrets: args.sandbox_image_pull_secrets, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: args.managed_ssh_ingress_enabled, + gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), + gateway_pod_selector: managed_ssh_gateway_pod_selector, + }, supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index c21ef8446d..d399d46508 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -798,7 +798,10 @@ impl ComputeRuntime { .await .map_err(|err| ComputeError::Message(err.to_string()))?; if driver.workspace_mode() == WorkspaceMode::Shared { - driver.backfill_gateway_id_labels().await; + driver + .backfill_gateway_id_labels() + .await + .map_err(|err| ComputeError::Message(err.to_string()))?; } let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); @@ -3817,7 +3820,18 @@ fn is_terminal_failure_reason(reason: &str) -> bool { #[cfg(test)] #[derive(Debug, Default)] -pub struct NoopTestDriver; +pub struct NoopTestDriver { + workspace_delete_failures: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl NoopTestDriver { + pub fn failing_workspace_deletes(count: usize) -> Self { + Self { + workspace_delete_failures: std::sync::atomic::AtomicUsize::new(count), + } + } +} #[cfg(test)] #[tonic::async_trait] @@ -3937,6 +3951,17 @@ impl ComputeDriver for NoopTestDriver { &self, _request: Request, ) -> Result, Status> { + if self + .workspace_delete_failures + .fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + return Err(Status::unavailable("injected workspace cleanup failure")); + } Ok(tonic::Response::new(DeleteWorkspaceResponse {})) } } @@ -3948,8 +3973,17 @@ pub async fn new_test_runtime(store: Arc) -> ComputeRuntime { #[cfg(test)] pub async fn new_test_runtime_for_driver(store: Arc, driver_name: &str) -> ComputeRuntime { + new_test_runtime_with_driver(store, driver_name, Arc::new(NoopTestDriver::default())).await +} + +#[cfg(test)] +pub async fn new_test_runtime_with_driver( + store: Arc, + driver_name: &str, + driver: Arc, +) -> ComputeRuntime { ComputeRuntime { - driver: TracedDriver::new(Arc::new(NoopTestDriver), "test".to_string()), + driver: TracedDriver::new(driver, "test".to_string()), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index a0ea195442..2c52acbe12 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -753,7 +753,9 @@ pub mod test_support { use crate::ServerState; use crate::auth::identity::{Identity, IdentityProvider}; use crate::auth::principal::{Principal, UserPrincipal}; - use crate::compute::{new_test_runtime, new_test_runtime_for_driver}; + use crate::compute::{ + NoopTestDriver, new_test_runtime, new_test_runtime_for_driver, new_test_runtime_with_driver, + }; use crate::persistence::Store; use crate::sandbox_index::SandboxIndex; use crate::sandbox_watch::SandboxWatchBus; @@ -813,6 +815,33 @@ pub mod test_support { None, )) } + + /// Build a test state whose compute driver fails the requested number of + /// workspace cleanup calls before succeeding. + pub async fn test_server_state_with_workspace_cleanup_failures( + failures: usize, + ) -> Arc { + let store = Arc::new( + Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(), + ); + crate::ensure_default_workspace(&store).await.unwrap(); + let driver = Arc::new(NoopTestDriver::failing_workspace_deletes(failures)); + let compute = new_test_runtime_with_driver(store.clone(), "test", driver).await; + Arc::new(ServerState::new( + Config::new(None) + .with_database_url("sqlite::memory:?cache=shared") + .with_credential_drivers(["test-static"]), + store, + compute, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + None, + )) + } } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index f2863511f1..d83ffab0e8 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -423,6 +423,15 @@ pub(super) async fn handle_delete_workspace( .await .map_err(|e| Status::internal(format!("delete workspace members failed: {e}")))?; + // Keep the terminating workspace durable until platform cleanup has been + // accepted. A failed cleanup can then be retried through this same path. + state.compute.delete_workspace(&name).await.map_err(|e| { + Status::new( + e.code(), + format!("delete workspace platform resources failed: {e}"), + ) + })?; + let deleted = state .store .delete_if(Workspace::object_type(), &ws_id, delete_version) @@ -435,10 +444,6 @@ pub(super) async fn handle_delete_workspace( } })?; - if deleted && let Err(e) = state.compute.delete_workspace(&name).await { - tracing::warn!(workspace = %name, error = %e, "failed to delete workspace platform resources"); - } - Ok(Response::new(DeleteWorkspaceResponse { deleted })) } @@ -620,7 +625,9 @@ mod tests { use openshell_core::proto::datamodel::v1::ObjectMeta; use tonic::{Code, Request}; - use crate::grpc::test_support::{authed_request, test_server_state}; + use crate::grpc::test_support::{ + authed_request, test_server_state, test_server_state_with_workspace_cleanup_failures, + }; #[tokio::test] async fn create_workspace_returns_metadata() { @@ -1326,6 +1333,62 @@ mod tests { assert!(resp.deleted); } + #[tokio::test] + async fn delete_workspace_retains_terminating_record_when_platform_cleanup_fails() { + let state = test_server_state_with_workspace_cleanup_failures(1).await; + + handle_create_workspace( + &state, + Request::new(CreateWorkspaceRequest { + name: "cleanup-retry".to_string(), + labels: HashMap::new(), + }), + ) + .await + .unwrap(); + + let err = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), Code::Unavailable); + + let retained: Workspace = state + .store + .get_message_by_name("", "cleanup-retry") + .await + .unwrap() + .expect("workspace must remain durable after cleanup failure"); + assert_ne!( + retained.metadata.unwrap().deletion_timestamp_ms, + 0, + "retained workspace must remain terminating" + ); + + let retry = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap() + .into_inner(); + assert!(retry.deleted); + assert!( + state + .store + .get_message_by_name::("", "cleanup-retry") + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn create_workspace_persists_labels_for_selector() { let state = test_server_state().await; diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 91008ff1d0..93dab354b6 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -177,7 +177,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | | imagePullSecrets | list | `[]` | Image pull secrets attached to gateway and helper pods. | | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | -| networkPolicy.enabled | bool | `true` | Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. | +| networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | | openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml index 9b8911fbe7..e9f88846c4 100644 --- a/deploy/helm/openshell/ci/values-workspace-managed.yaml +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -4,6 +4,8 @@ # E2E overlay: deploy the gateway in managed workspace mode. # Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. server: + sandboxImagePullSecrets: + - name: e2e-regcred drivers: kubernetes: workspaceMode: "managed" diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 0f62d8e915..a3b486c03c 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -101,4 +101,16 @@ rules: verbs: - create - get + {{- if .Values.networkPolicy.enabled }} + # Apply gateway-only SSH ingress isolation in managed namespaces. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - patch + - update + {{- end }} {{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index a8a3128e56..9d24dbd917 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -197,6 +197,11 @@ data: supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] + enabled = {{ .Values.networkPolicy.enabled }} + gateway_namespace = {{ .Release.Namespace | quote }} + gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } + [openshell.drivers.kubernetes.sidecar] proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index efaa117f44..e0bb908182 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -9,6 +9,29 @@ release: namespace: my-namespace tests: + - it: grants managed namespace NetworkPolicy apply permissions + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create", "patch", "update"] + + - it: omits managed NetworkPolicy permissions when isolation is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + networkPolicy.enabled: false + asserts: + - notContains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + any: true + - it: grants secrets patch and delete when credential driver is enabled (operator) set: server.drivers.kubernetes.workspaceMode: operator diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 784180fcb3..afacd01eb4 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -208,6 +208,13 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' + - it: configures managed SSH isolation with the gateway peer + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.managed_ssh_ingress\].*?enabled\s*=\s*true.*?gateway_namespace\s*=\s*"my-namespace".*?gateway_pod_selector\s*=.*?app\.kubernetes\.io/name.*?openshell' + - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 0b2fa1098c..33337c768e 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -403,7 +403,8 @@ server: # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. networkPolicy: - # -- Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. + # -- Restrict SSH ingress on sandbox pods to the gateway. In managed mode, + # the driver applies the equivalent policy to each workspace namespace. enabled: true # Built-in TLS PKI bootstrap via a pre-install/pre-upgrade hook Job. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index afb6ed6d9f..9e96ce0d1e 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -439,6 +439,7 @@ image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" + # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. supervisor_sideload_method = "image-volume" @@ -507,6 +508,11 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +[openshell.drivers.kubernetes.managed_ssh_ingress] +enabled = true +gateway_namespace = "openshell" +gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } + [openshell.drivers.kubernetes.sidecar] # UID used by relaxed long-running network sidecars. Strict process/binary-aware # sidecars run as UID 0 so Kubernetes grants the required /proc inspection @@ -520,6 +526,11 @@ proxy_uid = 1337 process_binary_aware_network_policy = true ``` +In managed workspace mode, the Kubernetes driver copies each explicitly named +`image_pull_secrets` Secret from `namespace` into the managed workspace +namespace on sandbox creation. Shared and operator modes require the Secret to +already exist in the sandbox namespace. + ### Docker Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 189f685ff3..26c9a0cb87 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -339,7 +339,8 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the gateway TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | -| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image pull secrets to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | +| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | +| `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 6bc0564ccf..a92562a6c9 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -11,8 +11,8 @@ //! //! Namespace cleanup after sandbox deletion is best-effort and depends on //! controller finalization timing. These tests focus on verifiable behavior: -//! namespace creation, labels, ServiceAccount provisioning, and sandbox CR -//! placement in the correct namespace. +//! namespace creation, labels, ServiceAccount and SSH NetworkPolicy +//! provisioning, and sandbox CR placement in the correct namespace. use std::process::Stdio; use std::time::Duration; @@ -178,6 +178,58 @@ async fn managed_creates_namespace_with_labels() { let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); + // The managed driver copies only explicitly configured image-pull Secrets + // from the gateway namespace into the workspace namespace. + let (ok, copied_secret) = kubectl(&[ + "get", + "secret", + "e2e-regcred", + "-n", + &ns, + "-o", + "jsonpath={.type}", + ]) + .await; + assert!( + ok && copied_secret.contains("kubernetes.io/dockerconfigjson"), + "configured image-pull Secret should be copied into {ns}: {copied_secret}" + ); + + // Verify SSH ingress is restricted to the gateway peer. Because Kubernetes + // NetworkPolicies are allowlists, the absence of a sandbox peer here + // denies sandbox-to-sandbox TCP 2222 traffic. + let (ok, policy) = kubectl(&[ + "get", + "networkpolicy", + "openshell-sandbox-ssh", + "-n", + &ns, + "-o", + "json", + ]) + .await; + assert!( + ok, + "managed SSH NetworkPolicy should exist in {ns}: {policy}" + ); + let policy: serde_json::Value = + serde_json::from_str(&policy).expect("managed SSH NetworkPolicy should be valid JSON"); + assert_eq!( + policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/managed-by"], + "openshell" + ); + assert_eq!(policy["spec"]["ingress"][0]["ports"][0]["port"], 2222); + assert_eq!( + policy["spec"]["ingress"][0]["from"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshell" + ); + assert!( + policy["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + ["app.kubernetes.io/name"] + .is_string(), + "SSH ingress peer must select gateway pods: {policy}" + ); + // Verify sandbox CR is in the managed namespace (not the gateway namespace). let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 01b6d855c7..f6d0efc4ec 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -814,6 +814,14 @@ else --wait --timeout 5m HELM_INSTALLED=1 + if [ -n "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET:-}" ]; then + kctl -n "${NAMESPACE}" create secret docker-registry \ + "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ + --docker-server=registry.example.test \ + --docker-username=e2e-user \ + --docker-password=e2e-password + fi + LOCAL_PORT="$(e2e_pick_port)" echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ diff --git a/tasks/test.toml b/tasks/test.toml index 0462851ed5..bef8baf2b5 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -172,7 +172,7 @@ run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-operator"] From 8e4073bab8a5b9b1970163126e6b1057e5943b18 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 13 Aug 2026 08:25:52 -0400 Subject: [PATCH 11/16] fix(k8s): address follow-up review findings Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 12 +++++ crates/openshell-driver-kubernetes/README.md | 10 ++++ .../openshell-driver-kubernetes/src/config.rs | 6 +++ .../openshell-driver-kubernetes/src/driver.rs | 25 +++++++-- .../openshell-driver-kubernetes/src/grpc.rs | 17 ++++++ crates/openshell-server/src/compute/mod.rs | 47 ++++++++++++---- .../helm/openshell/templates/clusterrole.yaml | 39 ++++++++++---- .../openshell/tests/clusterrole_test.yaml | 53 +++++++++++++++++-- docs/reference/sandbox-compute-drivers.mdx | 11 ++++ 9 files changed, 193 insertions(+), 27 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0786b0bcad..8c1a6ce9d8 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -375,6 +375,18 @@ per mode (`crates/openshell-server/src/auth/k8s_sa.rs`): `BTreeSet` populated by the label/file watchers. Starts empty (fail-closed) until the first watcher update. +These checks rely on an ownership invariant. In shared and managed modes, the +gateway and its trusted Agent Sandbox controller exclusively administer the +sandbox namespace, Sandbox CRs, sandbox pods, and configured sandbox +ServiceAccount. Other principals must not create or mutate those resources or +use that ServiceAccount. In operator mode, the platform operator retains +namespace lifecycle ownership, but must preserve the same exclusive control of +Sandbox CRs and the pods and ServiceAccount used for sandbox token bootstrap. +An allowlisted namespace is therefore a trust grant, not a tenant isolation +boundary. Kubernetes owner references alone do not prove which controller +created a pod, so admitting principals that can fabricate that resource chain +would allow them to claim an existing sandbox identity. + ### Credential Driver Integration The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`) diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 7f82454083..976ff184a9 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -16,6 +16,16 @@ workspace namespace modes via `workspace_mode`: (`operator_namespace_file`). Sandbox creation fails closed if the workspace namespace is not in the current allowlist. +Workspace namespace modes assume exclusive control of the sandbox identity +resource chain. In shared and managed modes, only the gateway and its trusted +Agent Sandbox controller may administer the sandbox namespace, Sandbox CRs, +sandbox pods, or configured sandbox ServiceAccount. In operator mode, the +platform operator owns namespace lifecycle but must prevent other principals +from creating or mutating Sandbox CRs, creating sandbox pods with fabricated +owner references, or using the configured sandbox ServiceAccount. Treat adding +a namespace to the operator allowlist as granting this trust; the allowlist is +not a tenant isolation boundary. + ## Runtime Model The gateway stores platform state and delegates sandbox workload creation to diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 04955acb70..8d31ed3679 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1857,6 +1857,12 @@ mod tests { validate_managed_namespace_name("gw1", "team-a").unwrap(); } + #[test] + fn validate_managed_namespace_name_rejects_invalid_workspace_characters() { + let err = validate_managed_namespace_name("gw1", "INVALID").unwrap_err(); + assert!(err.contains("not a valid DNS-1123 label")); + } + #[test] fn validate_managed_namespace_name_rejects_too_long() { let long_workspace = "a".repeat(50); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f9f0790e01..8bd61a5320 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -8,6 +8,7 @@ use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace, + validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ @@ -530,13 +531,19 @@ impl KubernetesComputeDriver { None }; - Ok(Self { + let driver = Self { client, watch_client, sandbox_api_version: Arc::new(OnceCell::new()), config, operator_allowlist, - }) + }; + + if driver.workspace_mode() == WorkspaceMode::Shared { + driver.backfill_gateway_id_labels().await?; + } + + Ok(driver) } pub fn capabilities(&self) -> Result { @@ -567,10 +574,21 @@ impl KubernetesComputeDriver { self.config.workspace_mode } + pub(crate) fn validate_workspace_namespace( + &self, + workspace: &str, + ) -> Result<(), KubernetesDriverError> { + if self.config.workspace_mode == WorkspaceMode::Managed { + validate_managed_namespace_name(&self.config.gateway_id, workspace) + .map_err(KubernetesDriverError::InvalidArgument)?; + } + Ok(()) + } + /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that /// predate its introduction. Runs once at startup in shared mode so that /// label-selector based lookups continue to find legacy resources. - pub async fn backfill_gateway_id_labels(&self) -> Result<(), KubernetesDriverError> { + async fn backfill_gateway_id_labels(&self) -> Result<(), KubernetesDriverError> { let sandbox_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await @@ -1333,6 +1351,7 @@ impl KubernetesComputeDriver { let name = sandbox.name.as_str(); let workspace = sandbox.workspace.as_str(); + self.validate_workspace_namespace(workspace)?; let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index cb8817c930..32c1ddf2a4 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -184,6 +184,9 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } + self.driver + .validate_workspace_namespace(&workspace) + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; match self.driver.workspace_mode() { WorkspaceMode::Managed => { self.driver @@ -213,6 +216,9 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } + self.driver + .validate_workspace_namespace(&workspace) + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; match self.driver.workspace_mode() { WorkspaceMode::Managed => { self.driver @@ -260,6 +266,17 @@ mod tests { assert_eq!(status.message(), "sandbox agent pod IP is not available"); } + #[test] + fn invalid_workspace_driver_errors_map_to_invalid_argument_status() { + let status: Status = ComputeDriverError::from(KubernetesDriverError::InvalidArgument( + "managed namespace is invalid".to_string(), + )) + .into(); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert_eq!(status.message(), "managed namespace is invalid"); + } + #[test] fn already_exists_driver_errors_map_to_already_exists_status() { let status: Status = ComputeDriverError::from(KubernetesDriverError::AlreadyExists).into(); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index d399d46508..9d2db23972 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -54,7 +54,7 @@ use openshell_driver_docker::DockerComputeDriver; #[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, - OperatorNamespaceAllowlist, WorkspaceMode, + OperatorNamespaceAllowlist, }; #[cfg(not(target_os = "windows"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; @@ -797,12 +797,6 @@ impl ComputeRuntime { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; - if driver.workspace_mode() == WorkspaceMode::Shared { - driver - .backfill_gateway_id_labels() - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - } let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); let runtime = Self::from_driver( @@ -895,26 +889,36 @@ impl ComputeRuntime { pub(crate) async fn ensure_workspace(&self, workspace: &str) -> Result<(), Status> { let workspace = workspace.to_string(); - self.driver + match self + .driver .call("driver.ensure_workspace", None, |driver| async move { driver .ensure_workspace(Request::new(EnsureWorkspaceRequest { workspace })) .await }) .await - .map(|_| ()) + { + Ok(_) => Ok(()), + Err(status) if status.code() == Code::Unimplemented => Ok(()), + Err(status) => Err(status), + } } pub(crate) async fn delete_workspace(&self, workspace: &str) -> Result<(), Status> { let workspace = workspace.to_string(); - self.driver + match self + .driver .call("driver.delete_workspace", None, |driver| async move { driver .delete_workspace(Request::new(DeleteWorkspaceRequest { workspace })) .await }) .await - .map(|_| ()) + { + Ok(_) => Ok(()), + Err(status) if status.code() == Code::Unimplemented => Ok(()), + Err(status) => Err(status), + } } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { @@ -4137,6 +4141,7 @@ mod tests { struct TestDriver { listed_sandboxes: Vec, current_sandboxes: Vec, + workspace_rpcs_unimplemented: bool, } #[tonic::async_trait] @@ -4255,6 +4260,9 @@ mod tests { &self, _request: Request, ) -> Result, Status> { + if self.workspace_rpcs_unimplemented { + return Err(Status::unimplemented("workspace lifecycle is unsupported")); + } Ok(tonic::Response::new(EnsureWorkspaceResponse {})) } @@ -4262,10 +4270,25 @@ mod tests { &self, _request: Request, ) -> Result, Status> { + if self.workspace_rpcs_unimplemented { + return Err(Status::unimplemented("workspace lifecycle is unsupported")); + } Ok(tonic::Response::new(DeleteWorkspaceResponse {})) } } + #[tokio::test] + async fn workspace_lifecycle_allows_legacy_driver_without_workspace_rpcs() { + let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: true, + ..Default::default() + })) + .await; + + runtime.ensure_workspace("legacy").await.unwrap(); + runtime.delete_workspace("legacy").await.unwrap(); + } + #[derive(Clone)] enum ControlledDeleteOutcome { Ok(bool), @@ -7624,6 +7647,7 @@ mod tests { #[tokio::test] async fn reconcile_store_with_backend_applies_driver_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: false, listed_sandboxes: vec![DriverSandbox { id: "sb-1".to_string(), name: "sandbox-a".to_string(), @@ -7811,6 +7835,7 @@ mod tests { #[tokio::test] async fn reconcile_store_with_backend_does_not_recreate_missing_record_from_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: false, listed_sandboxes: vec![DriverSandbox { id: "sb-1".to_string(), name: "sandbox-a".to_string(), diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index a3b486c03c..e8cd589f0e 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -73,12 +73,36 @@ rules: verbs: - get {{- end }} - {{- if ne $workspaceMode "shared" }} - # Secrets access for multi-namespace modes: - # - TLS sync uses server-side apply (patch) to copy the client TLS Secret - # into workspace namespaces so sandbox pods can mount mTLS material. - # - The kubernetes-secrets credential driver stores credentials as Secrets - # in workspace namespaces, additionally requiring delete for cleanup. + {{- $copiedSecretNames := list }} + {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} + {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} + {{- end }} + {{- if eq $workspaceMode "managed" }} + {{- range .Values.server.sandboxImagePullSecrets }} + {{- if .name }} + {{- $copiedSecretNames = append $copiedSecretNames .name }} + {{- end }} + {{- end }} + {{- end }} + {{- $copiedSecretNames = uniq $copiedSecretNames }} + {{- if $copiedSecretNames }} + # Copy only explicitly configured TLS and image-pull Secrets into workspace + # namespaces. Server-side apply authorizes these requests as patch operations. + - apiGroups: + - "" + resources: + - secrets + resourceNames: + {{- range $copiedSecretNames }} + - {{ . | quote }} + {{- end }} + verbs: + - get + - patch + {{- end }} + {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # The kubernetes-secrets credential driver uses dynamic hashed names in + # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. - apiGroups: - "" resources: @@ -86,11 +110,8 @@ rules: verbs: - get - create - - update - patch - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - delete - {{- end }} {{- end }} {{- if eq $workspaceMode "managed" }} # ServiceAccount creation in managed namespaces. diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index e0bb908182..bdc1e798a6 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -32,7 +32,7 @@ tests: resources: ["networkpolicies"] any: true - - it: grants secrets patch and delete when credential driver is enabled (operator) + - it: grants broad secret access when credential driver is enabled (operator) set: server.drivers.kubernetes.workspaceMode: operator server.credentialDrivers.kubernetesSecrets.enabled: true @@ -42,18 +42,63 @@ tests: content: apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "update", "patch", "delete"] + verbs: ["get", "create", "patch", "delete"] - - it: grants secrets patch for TLS sync even when credential driver is disabled (operator) + - it: restricts operator TLS sync to the configured secret set: server.drivers.kubernetes.workspaceMode: operator + server.tls.clientTlsSecretName: custom-client-tls asserts: - contains: path: rules content: apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "update", "patch"] + resourceNames: ["custom-client-tls"] + verbs: ["get", "patch"] + + - it: omits operator secret access when TLS and credential storage are disabled + set: + server.drivers.kubernetes.workspaceMode: operator + server.disableTls: true + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + any: true + + - it: restricts managed copies to TLS and configured image-pull secrets + set: + server.drivers.kubernetes.workspaceMode: managed + server.tls.clientTlsSecretName: custom-client-tls + server.sandboxImagePullSecrets: + - name: registry-one + - name: registry-two + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["custom-client-tls", "registry-one", "registry-two"] + verbs: ["get", "patch"] + + - it: restricts managed copies to image-pull secrets when TLS is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + server.disableTls: true + server.sandboxImagePullSecrets: + - name: registry-one + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["registry-one"] + verbs: ["get", "patch"] - it: omits secrets rule entirely in shared mode set: diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 26c9a0cb87..4f82c1e1a0 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -328,6 +328,17 @@ On hosts with restrictive firewalls (e.g. firewalld), the host firewall may addi Kubernetes-backed sandboxes run as pods in the configured sandbox namespace. Use Kubernetes for shared clusters, remote compute, GPU scheduling, and operator-managed environments. + +Kubernetes workspace namespaces are an administrative trust boundary. In +shared and managed modes, only the OpenShell gateway and its trusted Agent +Sandbox controller may administer Sandbox CRs, sandbox pods, or the configured +sandbox ServiceAccount in those namespaces. In operator mode, allowlist only +namespaces where the platform operator preserves that exclusive control. +Untrusted principals must not be able to create sandbox pods with fabricated +owner references or use the sandbox ServiceAccount. The operator namespace +allowlist is a trust grant, not a tenant isolation mechanism. + + Helm deployments set Kubernetes driver values through the chart. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). From 953eb42caaa3c12517fcf0e24a7638e093b3f99e Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 13 Aug 2026 17:20:42 -0400 Subject: [PATCH 12/16] fix(k8s): preserve workspace lookup after rebase Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 19 ++++++++++++++----- .../openshell-driver-kubernetes/src/grpc.rs | 7 +++---- crates/openshell-driver-podman/src/grpc.rs | 7 +++---- crates/openshell-driver-vm/src/driver.rs | 8 ++++---- crates/openshell-server/src/test_support.rs | 7 +++---- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 8bd61a5320..1437e3e14b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1560,14 +1560,13 @@ impl KubernetesComputeDriver { sandbox_id: &str, running: bool, ) -> Result<(AgentSandboxApi, String, String, Duration), String> { - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let list = tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api + lookup_api .api .list(&ListParams::default().labels(&selector)), ) @@ -1584,6 +1583,16 @@ impl KubernetesComputeDriver { .into_iter() .next() .ok_or_else(|| "sandbox not found".to_string())?; + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let agent_sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); let stop_timeout = kubernetes_sandbox_stop_timeout(&object); let kube_name = object .metadata diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 32c1ddf2a4..1643659a61 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -10,10 +10,9 @@ use openshell_core::proto::compute::v1::{ EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 79081e9419..19d0b55254 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -10,10 +10,9 @@ use openshell_core::proto::compute::v1::{ EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index a61ec11726..4dd2ea059b 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -45,10 +45,10 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, - WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 409d43180d..f8124ded6c 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -13,10 +13,9 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_server::ComputeDriver, - gateway_listener_requirement::Selector, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, }; use std::collections::HashMap; #[cfg(unix)] From c380e8d0993d0652c44cd148c461b5182ad00276 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 13 Aug 2026 18:46:22 -0400 Subject: [PATCH 13/16] fix(helm): allow managed secret creation Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 9 +++++++++ deploy/helm/openshell/templates/clusterrole.yaml | 12 ++++++++++++ deploy/helm/openshell/tests/clusterrole_test.yaml | 12 ++++++++++++ docs/reference/sandbox-compute-drivers.mdx | 7 +++++++ 4 files changed, 40 insertions(+) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 8c1a6ce9d8..42c53abc6a 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -341,6 +341,15 @@ DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-charac limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace `create`/`delete` and ServiceAccount `create`/`get` permissions. +Secret copies use server-side apply. Kubernetes authorizes an apply to an +existing Secret as `patch`, but also requires `create` authorization when the +target does not exist. RBAC cannot constrain `create` by `resourceNames`, so +managed mode grants cluster-wide Secret `create` while keeping source reads and +subsequent patches restricted to the explicitly configured TLS and image-pull +Secret names. The driver exercises `create` only in gateway-owned managed +namespaces. This depends on the managed-mode ownership invariant described +below; the gateway ServiceAccount must not be shared with unrelated workloads. + Operator mode does not create NetworkPolicies or copy image-pull Secrets. Platform teams must apply the gateway ingress boundary and provision configured image-pull Secrets in every operator-managed namespace. diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index e8cd589f0e..eb1ed8e1d0 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -99,6 +99,18 @@ rules: verbs: - get - patch + # Server-side apply uses PATCH for an existing Secret, but the API server + # additionally authorizes CREATE when the named Secret does not exist yet. + # Kubernetes RBAC cannot restrict CREATE by resourceNames because create + # authorization happens before the object name is available to RBAC. Keep + # reads and mutations name-restricted above; this broader grant is required + # only to create the initial copy in a gateway-owned managed namespace. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create {{- end }} {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} # The kubernetes-secrets credential driver uses dynamic hashed names in diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index bdc1e798a6..afecada9b6 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -84,6 +84,12 @@ tests: resources: ["secrets"] resourceNames: ["custom-client-tls", "registry-one", "registry-two"] verbs: ["get", "patch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] - it: restricts managed copies to image-pull secrets when TLS is disabled set: @@ -99,6 +105,12 @@ tests: resources: ["secrets"] resourceNames: ["registry-one"] verbs: ["get", "patch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] - it: omits secrets rule entirely in shared mode set: diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 4f82c1e1a0..897e780c39 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -371,6 +371,13 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +Managed-mode Secret copying requires the gateway ServiceAccount to create +Secrets. Kubernetes RBAC cannot restrict Secret `create` by resource name, so +the Helm chart grants cluster-wide Secret `create`; Secret `get` and `patch` +remain limited to the explicitly configured TLS and image-pull Secret names. +The driver creates copies only in gateway-owned managed namespaces. Do not +reuse the gateway ServiceAccount for unrelated workloads. + In `combined` topology, the agent container carries the Linux capabilities needed by the supervisor for network namespace setup, Landlock filesystem policy, process privilege changes, and network policy enforcement. In `sidecar` From a44ac68c4517be44a7e3677c3e2140a8081a5fe3 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 13 Aug 2026 22:09:06 -0400 Subject: [PATCH 14/16] fix(k8s): stop pods in workspace namespace Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 14 +++-- e2e/rust/tests/workspace_namespace_managed.rs | 51 +++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 1437e3e14b..8af605e412 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1497,11 +1497,11 @@ impl KubernetesComputeDriver { } pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), String> { - let (agent_sandbox_api, kube_name, pod_name, stop_timeout) = self + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; let legacy_pod_api = (agent_sandbox_api.resource.version == SANDBOX_VERSION_V1ALPHA1) - .then(|| Api::::namespaced(self.client.clone(), &self.config.namespace)); + .then(|| Api::::namespaced(self.client.clone(), &namespace)); let deadline = tokio::time::Instant::now() + stop_timeout; let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; @@ -1559,7 +1559,7 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, running: bool, - ) -> Result<(AgentSandboxApi, String, String, Duration), String> { + ) -> Result<(AgentSandboxApi, String, String, String, Duration), String> { let lookup_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await?; @@ -1634,7 +1634,13 @@ impl KubernetesComputeDriver { running, "Updated Kubernetes sandbox operating state" ); - Ok((agent_sandbox_api, kube_name, pod_name, stop_timeout)) + Ok(( + agent_sandbox_api, + kube_name, + pod_name, + namespace, + stop_timeout, + )) } pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index a92562a6c9..5be1972410 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -667,6 +667,57 @@ async fn managed_full_lifecycle_with_multiple_sandboxes() { } } +#[tokio::test] +async fn managed_stop_waits_for_workspace_pod_to_disappear() { + let ws = unique_workspace("mgdstop"); + let ns = managed_namespace(&ws); + let sandbox = "stop-sb"; + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![sandbox.into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + sandbox, + "--", + "echo", + "ready", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + + let (ok, pod_name) = kubectl(&[ + "get", + "sandbox", + sandbox, + "-n", + &ns, + "-o", + "jsonpath={.metadata.annotations.agents\\.x-k8s\\.io/pod-name}", + ]) + .await; + assert!(ok, "failed to resolve sandbox pod name: {pod_name}"); + let pod_name = pod_name.trim(); + assert!(!pod_name.is_empty(), "sandbox pod annotation was empty"); + + let (ok, out) = run_cli(&["sandbox", "stop", sandbox, "--workspace", &ws]).await; + assert!(ok, "sandbox stop failed: {out}"); + + let (exists, out) = kubectl(&["get", "pod", pod_name, "-n", &ns]).await; + assert!( + !exists, + "sandbox stop returned before workspace pod {pod_name} disappeared: {out}" + ); +} + #[tokio::test] async fn managed_rejects_invalid_dns1123_sandbox_name() { let ws = unique_workspace("mgddns"); From d2bf75f8b6e5502b3b6cbc32d4fc83b7427a9cd1 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 14 Aug 2026 10:22:13 -0400 Subject: [PATCH 15/16] test(k8s): scope pod deletion check to v1alpha1 Signed-off-by: Derek Carr --- e2e/rust/tests/workspace_namespace_managed.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 5be1972410..38de1da6a3 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -669,6 +669,23 @@ async fn managed_full_lifecycle_with_multiple_sandboxes() { #[tokio::test] async fn managed_stop_waits_for_workspace_pod_to_disappear() { + let (ok, sandbox_api_version) = kubectl(&[ + "get", + "crd", + "sandboxes.agents.x-k8s.io", + "-o", + "jsonpath={.spec.versions[?(@.storage==true)].name}", + ]) + .await; + assert!( + ok, + "failed to resolve Sandbox API version: {sandbox_api_version}" + ); + if sandbox_api_version.trim() != "v1alpha1" { + eprintln!("SKIP: legacy pod-disappearance fallback applies only to Sandbox API v1alpha1"); + return; + } + let ws = unique_workspace("mgdstop"); let ns = managed_namespace(&ws); let sandbox = "stop-sb"; From 66996589212c3f691eefe0f8db29692d7ba31085 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 14 Aug 2026 11:51:41 -0400 Subject: [PATCH 16/16] fix(k8s): address workspace namespace review findings Signed-off-by: Derek Carr --- .../src/lib.rs | 63 ++++++- crates/openshell-driver-kubernetes/README.md | 7 +- .../openshell-driver-kubernetes/src/config.rs | 74 ++++++-- .../openshell-driver-kubernetes/src/driver.rs | 178 +++++++++++++----- .../openshell-driver-kubernetes/src/grpc.rs | 62 +++--- .../openshell-driver-kubernetes/src/main.rs | 139 ++++++++------ crates/openshell-server/src/compute/mod.rs | 3 +- crates/openshell-server/src/lib.rs | 7 +- 8 files changed, 382 insertions(+), 151 deletions(-) diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index 1c59401cb1..91224212bd 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -532,12 +532,29 @@ impl KubernetesSecretsDriverSettings { } None => default_namespace(), }; + let gateway_id = config.gateway_id.unwrap_or_default(); + if config.workspace_mode == WorkspaceMode::Managed { + let gateway_id = trimmed_config_string("gateway_id", &gateway_id)?; + if !is_dns_label(gateway_id) { + return Err(Error::config( + "[openshell.credential_drivers.kubernetes-secrets] gateway_id must be a DNS-1123 label in managed workspace mode", + )); + } + // Workspace names are limited to 19 characters by the gateway. + // Keep the longest generated namespace within Kubernetes' 63-char + // DNS label limit, matching the Kubernetes compute driver. + if "openshell-".len() + gateway_id.len() + 1 + 19 > 63 { + return Err(Error::config( + "[openshell.credential_drivers.kubernetes-secrets] gateway_id is too long for managed workspace mode", + )); + } + } Ok(Self { namespace, allow_reference_namespace: config.allow_reference_namespace, workspace_mode: config.workspace_mode, - gateway_id: config.gateway_id.unwrap_or_default(), + gateway_id, }) } } @@ -808,6 +825,50 @@ mod tests { assert!(err.to_string().contains("namespace")); } + #[test] + fn settings_managed_mode_requires_gateway_id() { + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + }) + .unwrap_err(); + + assert!(err.to_string().contains("gateway_id")); + } + + #[test] + fn settings_managed_mode_rejects_invalid_gateway_id() { + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = "Invalid_Gateway" + }) + .unwrap_err(); + + assert!(err.to_string().contains("DNS-1123")); + } + + #[test] + fn settings_managed_mode_rejects_gateway_id_that_is_too_long() { + let gateway_id = "a".repeat(35); + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = gateway_id + }) + .unwrap_err(); + + assert!(err.to_string().contains("too long")); + } + + #[test] + fn settings_managed_mode_accepts_valid_gateway_id() { + let settings = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = "gateway-1" + }) + .unwrap(); + + assert_eq!(settings.gateway_id, "gateway-1"); + } + #[test] fn handle_resolves_secret_reference() { let reference = KubernetesSecretsCredentialDriver::parse_handle( diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 976ff184a9..ec37c4e67a 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -12,9 +12,12 @@ workspace namespace modes via `workspace_mode`: (`openshell-{gateway_id}-{workspace_name}`), creates a ServiceAccount in each, and copies OpenShift SCC annotations from the gateway namespace when present. - **Operator**: Workspace names map 1:1 to pre-provisioned namespaces discovered - via label selector (`operator_namespace_label`) and/or drop-in allowlist file + through exactly one source: either a label selector + (`operator_namespace_label`) or a drop-in allowlist file (`operator_namespace_file`). Sandbox creation fails closed if the workspace - namespace is not in the current allowlist. + namespace is not in the current allowlist. Workspace deletion only removes + gateway state; it never deletes or otherwise accesses the operator-managed + Kubernetes namespace. Workspace namespace modes assume exclusive control of the sandbox identity resource chain. In shared and managed modes, only the gateway and its trusted diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 8d31ed3679..bfa08a6642 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -849,6 +849,18 @@ pub struct OperatorNamespaceAllowlist { } impl OperatorNamespaceAllowlist { + fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { + self.inner + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + #[must_use] pub fn new() -> Self { Self { @@ -865,44 +877,35 @@ impl OperatorNamespaceAllowlist { /// Replace the entire allowlist (used by background watchers on refresh). pub fn replace(&self, new_set: BTreeSet) { - let mut guard = self.inner.write().expect("allowlist lock poisoned"); + let mut guard = self.write_guard(); *guard = new_set; } /// Merge additional namespaces into the allowlist. pub fn merge(&self, additional: &BTreeSet) { - let mut guard = self.inner.write().expect("allowlist lock poisoned"); + let mut guard = self.write_guard(); guard.extend(additional.iter().cloned()); } /// Read the current allowlist snapshot. pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { - self.inner.read().expect("allowlist lock poisoned") + self.read_guard() } /// Check whether a namespace is in the allowlist. #[must_use] pub fn contains(&self, namespace: &str) -> bool { - self.inner - .read() - .expect("allowlist lock poisoned") - .contains(namespace) + self.read_guard().contains(namespace) } /// Insert a namespace into the allowlist. Returns `true` if it was new. pub fn insert(&self, name: String) -> bool { - self.inner - .write() - .expect("allowlist lock poisoned") - .insert(name) + self.write_guard().insert(name) } /// Remove a namespace from the allowlist. Returns `true` if it was present. pub fn remove(&self, name: &str) -> bool { - self.inner - .write() - .expect("allowlist lock poisoned") - .remove(name) + self.write_guard().remove(name) } /// Return a clone of the inner `Arc` for sharing with background tasks. @@ -1674,6 +1677,18 @@ mod tests { ); } + #[test] + fn namespace_for_workspace_operator_requires_allowlist() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + + let err = cfg.namespace_for_workspace("prod", None).unwrap_err(); + assert_eq!(err, "operator mode requires a namespace allowlist"); + } + #[test] fn kube_resource_name_shared_prefixes_workspace() { let cfg = KubernetesComputeConfig::default(); @@ -1830,6 +1845,19 @@ mod tests { cfg.validate_workspace_mode().unwrap(); } + #[test] + fn validate_workspace_mode_operator_rejects_label_and_file() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + operator_namespace_file: Some("/etc/openshell/namespaces.json".to_string()), + ..KubernetesComputeConfig::default() + }; + + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("not both"), "{err}"); + } + #[test] fn dns_1123_label_validation() { assert!(is_dns_1123_label("openshell")); @@ -1885,4 +1913,20 @@ mod tests { al.replace(BTreeSet::new()); assert!(!al.contains("ns1")); } + + #[test] + fn operator_allowlist_recovers_from_poisoned_lock() { + let al = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["ns1".to_string()])); + let shared = al.shared(); + let _ = std::thread::spawn(move || { + let _guard = shared.write().unwrap(); + panic!("poison allowlist lock"); + }) + .join(); + + assert!(al.contains("ns1")); + assert!(al.insert("ns2".to_string())); + assert!(al.read().contains("ns2")); + assert!(al.remove("ns1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 8af605e412..ddc7fe2a4f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,8 +7,8 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace, - validate_managed_namespace_name, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, + managed_namespace, validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ @@ -53,7 +53,7 @@ use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use tokio::sync::{OnceCell, mpsc}; use tokio_stream::wrappers::ReceiverStream; use tracing::{debug, info, warn}; @@ -67,6 +67,8 @@ const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; pub enum KubernetesDriverError { #[error("sandbox already exists")] AlreadyExists, + #[error("sandbox not found")] + NotFound, #[error("{0}")] InvalidArgument(String), #[error("{0}")] @@ -88,6 +90,7 @@ impl From for openshell_core::ComputeDriverError { fn from(err: KubernetesDriverError) -> Self { match err { KubernetesDriverError::AlreadyExists => Self::AlreadyExists, + KubernetesDriverError::NotFound => Self::NotFound, KubernetesDriverError::InvalidArgument(m) => Self::InvalidArgument(m), KubernetesDriverError::Precondition(m) => Self::Precondition(m), KubernetesDriverError::Message(m) => Self::Message(m), @@ -354,23 +357,13 @@ fn validate_kubernetes_driver_volume_mounts( Ok(()) } -// TODO: replace with an openshell_core Kubernetes-name helper once available. -fn is_dns_label(label: &str) -> bool { - if label.is_empty() || label.len() > 63 || label.starts_with('-') || label.ends_with('-') { - return false; - } - label - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') -} - // TODO: replace with an openshell_core Kubernetes-name helper once available. fn is_dns_subdomain(value: &str) -> bool { - value.len() <= 253 && value.split('.').all(is_dns_label) + value.len() <= 253 && value.split('.').all(is_dns_1123_label) } fn validate_kubernetes_dns1123_label(value: &str, field: &str) -> Result<(), String> { - if !is_dns_label(value) { + if !is_dns_1123_label(value) { return Err(format!( "{field} must be a DNS-1123 label: use lowercase alphanumeric characters or '-', start and end with an alphanumeric character, and use at most 63 characters" )); @@ -474,7 +467,10 @@ impl std::fmt::Debug for KubernetesComputeDriver { } impl KubernetesComputeDriver { - pub async fn new(config: KubernetesComputeConfig) -> Result { + pub async fn new( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + ) -> Result { config .validate_workspace_mode() .map_err(KubernetesDriverError::Precondition)?; @@ -519,11 +515,12 @@ impl KubernetesComputeDriver { watch_client.clone(), label.clone(), allowlist.clone(), + shutdown_rx.clone(), ); } if let Some(ref path) = config.operator_namespace_file { - spawn_namespace_file_watcher(path.into(), allowlist.clone()); + spawn_namespace_file_watcher(path.into(), allowlist.clone(), shutdown_rx.clone()); } Some(allowlist) @@ -660,15 +657,20 @@ impl KubernetesComputeDriver { let ns_name = managed_namespace(&self.config.gateway_id, workspace); let ns_api: Api = Api::all(self.client.clone()); - let gateway_ns_api: Api = Api::all(self.client.clone()); let gateway_ns_annotations = match tokio::time::timeout( KUBE_API_TIMEOUT, - gateway_ns_api.get(&self.config.namespace), + ns_api.get(&self.config.namespace), ) .await { Ok(Ok(ns)) => ns.metadata.annotations.unwrap_or_default(), - _ => BTreeMap::new(), + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout getting gateway namespace {} for SCC annotations", + self.config.namespace + ))); + } }; let mut labels = BTreeMap::new(); @@ -1496,7 +1498,7 @@ impl KubernetesComputeDriver { } } - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), String> { + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; @@ -1508,10 +1510,10 @@ impl KubernetesComputeDriver { loop { let now = tokio::time::Instant::now(); if now >= deadline { - return Err(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes sandbox to stop", stop_timeout.as_secs() - )); + ))); } let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); let object = tokio::time::timeout( @@ -1520,36 +1522,38 @@ impl KubernetesComputeDriver { ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API while checking sandbox stop", request_timeout.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; if kubernetes_sandbox_has_stopped_condition(&object) { return Ok(()); } if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(error); + return Err(KubernetesDriverError::Message(error)); } if let Some(pod_api) = legacy_pod_api.as_ref() - && kubernetes_sandbox_pod_is_gone(pod_api, &pod_name, deadline).await? + && kubernetes_sandbox_pod_is_gone(pod_api, &pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)? { return Ok(()); } let now = tokio::time::Instant::now(); if now >= deadline { - return Err(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes sandbox to stop", stop_timeout.as_secs() - )); + ))); } tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; poll_interval = next_stop_poll_interval(poll_interval); } } - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), String> { + pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { self.patch_sandbox_operating_state(sandbox_id, true) .await .map(|_| ()) @@ -1559,10 +1563,11 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, running: bool, - ) -> Result<(AgentSandboxApi, String, String, String, Duration), String> { + ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { let lookup_api = self .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; + .await + .map_err(KubernetesDriverError::Message)?; let selector = self.sandbox_lookup_selector(sandbox_id); let list = tokio::time::timeout( KUBE_API_TIMEOUT, @@ -1572,17 +1577,17 @@ impl KubernetesComputeDriver { ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; let object = list .items .into_iter() .next() - .ok_or_else(|| "sandbox not found".to_string())?; + .ok_or(KubernetesDriverError::NotFound)?; let namespace = object .metadata .namespace @@ -1594,10 +1599,9 @@ impl KubernetesComputeDriver { &namespace, ); let stop_timeout = kubernetes_sandbox_stop_timeout(&object); - let kube_name = object - .metadata - .name - .ok_or_else(|| "sandbox resource has no name".to_string())?; + let kube_name = object.metadata.name.ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; let pod_name = object .metadata .annotations @@ -1621,12 +1625,12 @@ impl KubernetesComputeDriver { ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; info!( sandbox_id, @@ -4292,17 +4296,34 @@ fn spawn_namespace_label_watcher( client: Client, label_selector: String, allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, ) { let ns_api: Api = Api::all(client); let watcher_config = watcher::Config::default().labels(&label_selector); + let jitter_seed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_secs() ^ u64::from(duration.subsec_nanos()) + }); tokio::spawn(async move { + let mut retry_attempt = 0; loop { let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); loop { - match stream.try_next().await { + let event = tokio::select! { + result = stream.try_next() => result, + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + match event { Ok(Some(Event::Applied(ns))) => { + retry_attempt = 0; if let Some(name) = ns.metadata.name.as_deref() && allowlist.insert(name.to_string()) { @@ -4310,6 +4331,7 @@ fn spawn_namespace_label_watcher( } } Ok(Some(Event::Deleted(ns))) => { + retry_attempt = 0; if let Some(name) = ns.metadata.name.as_deref() && allowlist.remove(name) { @@ -4320,6 +4342,7 @@ fn spawn_namespace_label_watcher( } } Ok(Some(Event::Restarted(namespaces))) => { + retry_attempt = 0; let names: std::collections::BTreeSet = namespaces .into_iter() .filter_map(|ns| ns.metadata.name) @@ -4342,7 +4365,17 @@ fn spawn_namespace_label_watcher( } } - tokio::time::sleep(Duration::from_secs(2)).await; + let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); + warn!(?retry_delay, "operator namespace watcher reconnecting"); + tokio::select! { + () = tokio::time::sleep(retry_delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + retry_attempt = retry_attempt.saturating_add(1); } }); @@ -4352,6 +4385,15 @@ fn spawn_namespace_label_watcher( ); } +fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { + let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); + let max_jitter_secs = base_secs / 4; + let mixed_seed = + jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let jitter_secs = mixed_seed % (max_jitter_secs + 1); + Duration::from_secs(base_secs + jitter_secs) +} + fn load_namespace_file(path: &Path) -> Result, String> { let contents = std::fs::read_to_string(path) .map_err(|e| format!("failed to read {}: {e}", path.display()))?; @@ -4360,7 +4402,11 @@ fn load_namespace_file(path: &Path) -> Result Ok(names.into_iter().collect()) } -fn spawn_namespace_file_watcher(path: PathBuf, allowlist: OperatorNamespaceAllowlist) { +fn spawn_namespace_file_watcher( + path: PathBuf, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { match load_namespace_file(&path) { Ok(names) => { let count = names.len(); @@ -4428,7 +4474,15 @@ fn spawn_namespace_file_watcher(path: PathBuf, allowlist: OperatorNamespaceAllow ); loop { - let got_event = rx.recv().await.is_some(); + let got_event = tokio::select! { + event = rx.recv() => event.is_some(), + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; if !got_event { warn!("operator namespace file watcher disconnected"); break; @@ -4462,6 +4516,11 @@ fn spawn_namespace_file_watcher(path: PathBuf, allowlist: OperatorNamespaceAllow warn!("operator namespace file watcher disconnected"); return; } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } } } } @@ -7824,4 +7883,27 @@ mod tests { Some("namespace-uid".to_string()) ); } + + #[test] + fn namespace_watcher_retry_delay_is_bounded_exponential_with_jitter() { + let seed = 42; + let expected_ranges = [(2, 2), (4, 5), (8, 10), (16, 20), (24, 30), (24, 30)]; + + for (attempt, (minimum, maximum)) in expected_ranges.into_iter().enumerate() { + let attempt = u32::try_from(attempt).unwrap(); + let delay = namespace_watcher_retry_delay(attempt, seed).as_secs(); + assert!( + (minimum..=maximum).contains(&delay), + "attempt {attempt} produced {delay}s" + ); + } + } + + #[test] + fn namespace_watcher_retry_delay_uses_seeded_jitter() { + assert_ne!( + namespace_watcher_retry_delay(3, 1), + namespace_watcher_retry_delay(3, 2) + ); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 1643659a61..ce23cf0189 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -124,7 +124,7 @@ impl ComputeDriver for ComputeDriverService { self.driver .stop_sandbox(&request.sandbox_id) .await - .map_err(kubernetes_lifecycle_status)?; + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; Ok(Response::new(StopSandboxResponse {})) } @@ -139,7 +139,7 @@ impl ComputeDriver for ComputeDriverService { self.driver .start_sandbox(&request.sandbox_id) .await - .map_err(kubernetes_lifecycle_status)?; + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; Ok(Response::new(StartSandboxResponse {})) } @@ -215,41 +215,26 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } - self.driver - .validate_workspace_namespace(&workspace) - .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; - match self.driver.workspace_mode() { - WorkspaceMode::Managed => { - self.driver - .delete_namespace(&workspace) - .await - .map_err(|e| Status::internal(e.to_string()))?; - } - WorkspaceMode::Operator => { - if let Some(allowlist) = self.driver.operator_allowlist() - && !allowlist.contains(&workspace) - { - return Err(Status::permission_denied(format!( - "workspace '{workspace}' is not in the operator namespace allowlist" - ))); - } - } - WorkspaceMode::Shared => {} + if workspace_delete_requires_namespace_access(self.driver.workspace_mode()) { + self.driver + .validate_workspace_namespace(&workspace) + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; + self.driver + .delete_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; } Ok(Response::new(DeleteWorkspaceResponse {})) } } -fn kubernetes_lifecycle_status(message: String) -> Status { - if message == "sandbox not found" { - Status::not_found(message) - } else { - Status::internal(message) - } +fn workspace_delete_requires_namespace_access(mode: WorkspaceMode) -> bool { + matches!(mode, WorkspaceMode::Managed) } #[cfg(test)] mod tests { + use super::{WorkspaceMode, workspace_delete_requires_namespace_access}; use crate::KubernetesDriverError; use openshell_core::ComputeDriverError; use tonic::Status; @@ -283,4 +268,25 @@ mod tests { assert_eq!(status.code(), tonic::Code::AlreadyExists); assert_eq!(status.message(), "sandbox already exists"); } + + #[test] + fn not_found_driver_errors_map_to_not_found_status() { + let status: Status = ComputeDriverError::from(KubernetesDriverError::NotFound).into(); + + assert_eq!(status.code(), tonic::Code::NotFound); + assert_eq!(status.message(), "sandbox not found"); + } + + #[test] + fn only_managed_workspace_delete_accesses_the_namespace() { + assert!(workspace_delete_requires_namespace_access( + WorkspaceMode::Managed + )); + assert!(!workspace_delete_requires_namespace_access( + WorkspaceMode::Operator + )); + assert!(!workspace_delete_requires_namespace_access( + WorkspaceMode::Shared + )); + } } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 75d56245c7..30b4fcada9 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -179,6 +179,29 @@ struct Args { sandbox_gid: Option, } +async fn shutdown_signal() { + #[cfg(unix)] + { + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = terminate => {} + } + } + + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); @@ -201,67 +224,75 @@ async fn main() -> Result<()> { }) .collect::>>()?; - let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { - workspace_mode: args.workspace_mode, - gateway_id: args.gateway_id, - namespace: args.sandbox_namespace, - operator_namespace_label: args.operator_namespace_label, - operator_namespace_file: args.operator_namespace_file, - service_account_name: args.sandbox_service_account, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), - image_pull_secrets: args.sandbox_image_pull_secrets, - managed_ssh_ingress: ManagedSshIngressConfig { - enabled: args.managed_ssh_ingress_enabled, - gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), - gateway_pod_selector: managed_ssh_gateway_pod_selector, - }, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let driver = KubernetesComputeDriver::new( + KubernetesComputeConfig { + workspace_mode: args.workspace_mode, + gateway_id: args.gateway_id, + namespace: args.sandbox_namespace, + operator_namespace_label: args.operator_namespace_label, + operator_namespace_file: args.operator_namespace_file, + service_account_name: args.sandbox_service_account, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_secrets: args.sandbox_image_pull_secrets, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: args.managed_ssh_ingress_enabled, + gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), + gateway_pod_selector: managed_ssh_gateway_pod_selector, + }, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_sideload_method: args.supervisor_sideload_method, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args + .sidecar_process_binary_aware_network_policy, + }, + https_proxy: args.https_proxy, + no_proxy: args.no_proxy, + proxy_auth_secret_name: args.proxy_auth_secret_name, + proxy_auth_secret_key: args.proxy_auth_secret_key, + proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + ssh_socket_path: args.sandbox_ssh_socket_path, + client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), + host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), + enable_user_namespaces: args.enable_user_namespaces, + app_armor_profile: args.app_armor_profile, + workspace_default_storage_size: std::env::var( + "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", + ) + .unwrap_or_else(|_| { + openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() + }), + workspace_storage_class: std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") + .unwrap_or_default(), + default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") + .unwrap_or_default(), + sa_token_ttl_secs: args.sa_token_ttl_secs, + provider_spiffe_workload_api_socket_path: args + .provider_spiffe_workload_api_socket_path + .unwrap_or_default(), + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, }, - https_proxy: args.https_proxy, - no_proxy: args.no_proxy, - proxy_auth_secret_name: args.proxy_auth_secret_name, - proxy_auth_secret_key: args.proxy_auth_secret_key, - proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), - proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_socket_path: args.sandbox_ssh_socket_path, - client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), - host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), - enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, - workspace_default_storage_size: std::env::var( - "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", - ) - .unwrap_or_else(|_| { - openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() - }), - workspace_storage_class: std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") - .unwrap_or_default(), - default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") - .unwrap_or_default(), - sa_token_ttl_secs: args.sa_token_ttl_secs, - provider_spiffe_workload_api_socket_path: args - .provider_spiffe_workload_api_socket_path - .unwrap_or_default(), - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) + shutdown_rx, + ) .await .into_diagnostic()?; info!(address = %args.bind_address, "Starting Kubernetes compute driver"); tonic::transport::Server::builder() .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve(args.bind_address) + .serve_with_shutdown(args.bind_address, async move { + shutdown_signal().await; + let _ = shutdown_tx.send(true); + }) .await .into_diagnostic() } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 9d2db23972..e09b63de09 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -793,8 +793,9 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, + shutdown_rx: watch::Receiver, ) -> Result<(Self, Option), ComputeError> { - let driver = KubernetesComputeDriver::new(config) + let driver = KubernetesComputeDriver::new(config, shutdown_rx) .await .map_err(|err| ComputeError::Message(err.to_string()))?; let operator_allowlist_arc = driver.operator_allowlist().cloned(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index dfc2645d90..dbdfb1ac43 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -284,6 +284,7 @@ pub(crate) async fn run_server( config_file, guest_tls, } = startup; + let (shutdown_tx, shutdown_rx) = watch::channel(false); auth::descriptor_authz::init() .map_err(|error| Error::config(format!("invalid gRPC authorization metadata: {error}")))?; @@ -357,6 +358,7 @@ pub(crate) async fn run_server( sandbox_watch_bus.clone(), tracing_log_bus.clone(), supervisor_sessions.clone(), + shutdown_rx.clone(), ) .await?; let gateway_interceptors = @@ -501,8 +503,6 @@ pub(crate) async fn run_server( let state = Arc::new(state); - let (shutdown_tx, shutdown_rx) = watch::channel(false); - // Start sandboxes that were stopped during the previous gateway // shutdown so the running compute state matches the persisted store. // Runs before watchers spawn so the watch loop sees the post-start @@ -872,6 +872,7 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com #[allow(clippy::too_many_arguments)] type OperatorAllowlistArc = Option; +#[allow(clippy::too_many_arguments)] async fn build_compute_runtime( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, @@ -880,6 +881,7 @@ async fn build_compute_runtime( sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, + shutdown_rx: watch::Receiver, ) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { let driver = configured_compute_driver(config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); @@ -903,6 +905,7 @@ async fn build_compute_runtime( sandbox_watch_bus, tracing_log_bus, supervisor_sessions.clone(), + shutdown_rx, ) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;