From 1a00ca133d598049e57e53177155052efa15912a Mon Sep 17 00:00:00 2001 From: Itamar Zand <133867530+ItamarZand88@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:11:47 +0300 Subject: [PATCH 1/2] ci(sandbox): run the egressDeny guard weekly instead of never The one test that proves a sandbox declared egressDeny cannot reach the internet is `#[ignore]`d and named by no workflow, so it runs when someone remembers. A weekly job now builds the probe image, renders the stack with the shipped emitter, runs the check against a real account and removes everything. The test stays `#[ignore]`d and is named exactly by the workflow, with `--no-tests=fail`. Gating it on the probe environment instead would report a provisioning failure as a pass, which is the hole this closes. Teardown runs as a trailing step and again as a dependent job: a cancelled run gives `always()` only whatever window the runner allows, which does not reliably cover deleting a VPC. --- .github/workflows/egress-deny-guard.yml | 325 ++++++++++++++++++ .../src/aws/lambda_microvms.rs | 172 ++++++++- crates/alien-build/examples/sandbox-bundle.rs | 32 ++ scripts/README.md | 6 + scripts/egress-deny-guard-teardown.sh | 114 ++++++ 5 files changed, 642 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/egress-deny-guard.yml create mode 100644 crates/alien-build/examples/sandbox-bundle.rs create mode 100755 scripts/egress-deny-guard-teardown.sh diff --git a/.github/workflows/egress-deny-guard.yml b/.github/workflows/egress-deny-guard.yml new file mode 100644 index 000000000..8e834046b --- /dev/null +++ b/.github/workflows/egress-deny-guard.yml @@ -0,0 +1,325 @@ +name: Egress Deny Guard + +# The only automated check that a sandbox declared `egressDeny` cannot reach the internet. +# Weekly rather than per-PR: it needs a real account and a MicroVM image build, and what it +# guards against is an AWS-side behaviour change rather than a change in this repository. + +on: + schedule: + - cron: "43 4 * * 2" + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: egress-deny-guard + cancel-in-progress: false + +env: + STACK_NAME: egress-deny-guard + # Docker base the bundle builds on. AL2023 matches the MicroVM base image the emitter names and + # ships the curl the probe runs. + SANDBOX_BASE_IMAGE: public.ecr.aws/amazonlinux/amazonlinux:2023 + AGENT_TARGET: aarch64-unknown-linux-musl + # Fall back to local compilation when the shared sccache backend returns a + # transient error (e.g. webdav 403) instead of failing the whole build. + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + +jobs: + egress-deny: + name: egressDeny is enforced + runs-on: depot-ubuntu-24.04-arm-8 + timeout-minutes: 90 + # Makes the OIDC `sub` `repo:OWNER/REPO:environment:egress-deny-guard` whatever branch a + # manual run starts from, so one trust policy covers the schedule and a deliberate break. + environment: egress-deny-guard + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Configure git credentials + run: git config --global url."https://x-access-token:${{ secrets.REPO_ACCESS_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - uses: dtolnay/rust-toolchain@7c8d7d138f5c09cef361f8214cf96882cd029cdb # nightly + with: + toolchain: nightly + targets: aarch64-unknown-linux-musl + + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + with: + version: v0.16.0 + + - uses: taiki-e/install-action@43cb5d9d3c33252b8482ffa34f4e609859a530d8 # cargo-nextest + with: + tool: cargo-nextest + + - name: Install protoc + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 + with: + version: "27.x" + repo-token: ${{ github.token }} + + - name: Build the sandbox agent + run: | + set -euo pipefail + cargo build -p alien-sandbox-agent --release --target "$AGENT_TARGET" + agent="target/$AGENT_TARGET/release/alien-sandbox-agent" + # A MicroVM image accepts aarch64 only, and the wrong architecture surfaces minutes + # later as an image that never becomes active rather than as a build failure. + file "$agent" + file "$agent" | grep -q "ARM aarch64" + echo "AGENT_BINARY=$agent" >> "$GITHUB_ENV" + + - name: Write the image bundle + run: | + set -euo pipefail + mkdir -p .guard + cargo run -p alien-build --example sandbox-bundle -- \ + "$AGENT_BINARY" "$SANDBOX_BASE_IMAGE" .guard/sandbox-bundle.zip + # Per-run key, so nothing collides. A cancelled run cannot remove its own object and + # no later run can guess the key: expiry on the `egress-deny-guard/` prefix is what + # reclaims it, and that lives in the bucket's lifecycle configuration. + echo "BUNDLE_KEY=egress-deny-guard/${{ github.run_id }}-${{ github.run_attempt }}.zip" \ + >> "$GITHUB_ENV" + + - name: Render the sandbox stack + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: | + set -euo pipefail + cat > .guard/alien.json < .guard/stack-settings.yaml <<'YAML' + network: + type: create + availabilityZones: 2 + YAML + # Rendered by the shipped emitter rather than written out here: the deny is a + # loopback-only egress rule on the connector's security group, and a hand-written copy + # would guard the copy instead of what deployments get. + cargo run -p alien-cli --bin alien -- render \ + --format cloudformation \ + --stack .guard/alien.json \ + --stack-settings .guard/stack-settings.yaml \ + --registration-mode outputs \ + --output .guard + + - name: Build the guard test + # Built by nextest, and before credentials are assumed: nextest compiles its own test + # binaries, and doing that after the assume spends the session on compilation. + run: cargo nextest run -p alien-aws-clients --lib --no-run + + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + # The job budget is 90 minutes and a role's default ceiling is one hour, which would + # expire mid-test and fail on signing rather than on the property under test. + role-duration-seconds: 5400 + + - name: Resolve the target account + run: | + set -euo pipefail + account_id=$(aws sts get-caller-identity --query Account --output text) + echo "::add-mask::$account_id" + echo "ACCOUNT_ID=$account_id" >> "$GITHUB_ENV" + + - name: Remove anything an earlier run left behind + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" + + - name: Upload the image bundle + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: aws s3 cp .guard/sandbox-bundle.zip "s3://$ARTIFACT_BUCKET/$BUNDLE_KEY" + + - name: Deploy the sandbox stack + env: + MANAGING_ROLE_ARN: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + run: | + set -euo pipefail + aws cloudformation deploy \ + --template-file .guard/template.yaml \ + --stack-name "$STACK_NAME" \ + --no-fail-on-empty-changeset \ + --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \ + --parameter-overrides \ + "ManagingRoleArn=$MANAGING_ROLE_ARN" \ + "ManagingAccountId=$ACCOUNT_ID" + + - name: Read the probe image and connector + run: | + set -euo pipefail + resources=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='DeploymentResources'].OutputValue" \ + --output text) + image_arn=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.imageArn // empty' | head -1) + image_version=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.imageVersion // empty' | head -1) + connector=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.egressConnectorArns[0] // empty' | head -1) + echo "::add-mask::$image_arn" + echo "::add-mask::$connector" + + # RunMicrovm refuses a bare name with "Malformed ARN", which reads like a broken image + # rather than the wrong identifier having been read out of the stack. + case "$image_arn" in + arn:*) ;; + *) echo "::error::the stack's sandbox output is not an image ARN"; exit 1 ;; + esac + case "$connector" in + arn:*) ;; + *) echo "::error::the stack produced no egress connector ARN"; exit 1 ;; + esac + if [ -z "$image_version" ]; then + echo "::error::the probe image reports no active version" + exit 1 + fi + + { + echo "PROBE_IMAGE_NAME=$image_arn" + echo "PROBE_IMAGE_VERSION=$image_version" + echo "PROBE_CONNECTOR_ARN=$connector" + } >> "$GITHUB_ENV" + + - name: A denied sandbox cannot reach the internet + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: | + set -euo pipefail + # No --no-capture: nextest prints a failing test's output anyway, and streaming the + # passing run puts the session endpoint in a world-readable log. + if ! cargo nextest run -p alien-aws-clients --lib \ + --no-tests=fail --run-ignored=all \ + -E 'test(=aws::lambda_microvms::live_deny::a_denied_sandbox_cannot_reach_the_internet_and_an_open_one_can)' + then + echo "::error title=egressDeny regression::a sandbox declared egressDeny was not denied egress, or the guard could not prove it — the assertion above says which" + exit 1 + fi + + - name: Refresh credentials for teardown + if: always() + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + # The session above may have expired inside the job; take the web-identity path again + # rather than inheriting it. + unset-current-credentials: true + + - name: Tear down + if: always() + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" + + - name: Remove the uploaded bundle + if: always() && env.BUNDLE_KEY != '' + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: aws s3 rm "s3://$ARTIFACT_BUCKET/$BUNDLE_KEY" + + # Cleanup safety net. The teardown above is a step in the job it cleans up after, so a timeout + # or a cancel gives it only whatever window the runner allows — which does not reliably cover + # deleting a VPC. This job has its own timeout and its own credentials. + # + # It checks with the CLI first and only builds when something actually survived: a cancelled run + # leaves a MicroVM and an image version holding the image open, no stock `aws` subcommand + # terminates either, and a delete on a held image is accepted while removing nothing — so + # `delete-stack` alone cannot be trusted to clear it. + cleanup: + needs: [egress-deny] + if: always() + runs-on: depot-ubuntu-24.04-arm-8 + timeout-minutes: 45 + environment: egress-deny-guard + steps: + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + id: creds + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + mask-aws-account-id: true + + - name: Did anything survive? + id: survived + run: | + set -uo pipefail + if ! err=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>&1 >/dev/null); then + case "$err" in + *"does not exist"*) echo "nothing left behind"; echo "stack=absent" >> "$GITHUB_OUTPUT"; exit 0 ;; + *) echo "::error::describe-stacks was inconclusive: $err"; exit 1 ;; + esac + fi + echo "::warning::$STACK_NAME outlived its job; reclaiming it" + echo "stack=present" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: steps.survived.outputs.stack == 'present' + + - name: Configure git credentials + if: steps.survived.outputs.stack == 'present' + run: git config --global url."https://x-access-token:${{ secrets.REPO_ACCESS_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - uses: dtolnay/rust-toolchain@7c8d7d138f5c09cef361f8214cf96882cd029cdb # nightly + if: steps.survived.outputs.stack == 'present' + with: + toolchain: nightly + + - uses: taiki-e/install-action@43cb5d9d3c33252b8482ffa34f4e609859a530d8 # cargo-nextest + if: steps.survived.outputs.stack == 'present' + with: + tool: cargo-nextest + + - name: Install protoc + if: steps.survived.outputs.stack == 'present' + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 + with: + version: "27.x" + repo-token: ${{ github.token }} + + - name: Reclaim and remove the stack + if: steps.survived.outputs.stack == 'present' + env: + # The account id is a plain step output; the key, secret and token are exported to the + # environment instead and are only outputs under `output-credentials`. + AWS_TARGET_ACCOUNT_ID: ${{ steps.creds.outputs.aws-account-id }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" diff --git a/crates/alien-aws-clients/src/aws/lambda_microvms.rs b/crates/alien-aws-clients/src/aws/lambda_microvms.rs index fac41e8f6..ba197f2c6 100644 --- a/crates/alien-aws-clients/src/aws/lambda_microvms.rs +++ b/crates/alien-aws-clients/src/aws/lambda_microvms.rs @@ -992,12 +992,21 @@ mod live_deny { fn client() -> LambdaMicrovmsClient { let root: StdPathBuf = workspace_root::get_workspace_root(); dotenvy::from_path(root.join(".env.test")).ok(); + // Empty is unset, not configured: a workflow step reading an expression that never + // resolved passes "" rather than nothing, and signing with it surfaces a 403 instead of + // saying the credentials were never there. + let required = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| panic!("{name} must be set and non-empty")) + }; let config = crate::AwsClientConfig { - account_id: std::env::var("AWS_TARGET_ACCOUNT_ID").expect("AWS_TARGET_ACCOUNT_ID"), - region: std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"), + account_id: required("AWS_TARGET_ACCOUNT_ID"), + region: required("AWS_TARGET_REGION"), credentials: AwsCredentials::AccessKeys { - access_key_id: std::env::var("AWS_TARGET_ACCESS_KEY_ID").expect("key"), - secret_access_key: std::env::var("AWS_TARGET_SECRET_ACCESS_KEY").expect("secret"), + access_key_id: required("AWS_TARGET_ACCESS_KEY_ID"), + secret_access_key: required("AWS_TARGET_SECRET_ACCESS_KEY"), session_token: std::env::var("AWS_TARGET_SESSION_TOKEN") .ok() .filter(|token| !token.is_empty()), @@ -1123,6 +1132,11 @@ mod live_deny { } } + /// The only automated check that `egressDeny` denies, run weekly by `egress-deny-guard`. + /// + /// Left `#[ignore]`d and named exactly by that workflow rather than gated on the probe + /// environment: a gate that skipped when the image was missing would report a provisioning + /// failure as a pass, which is the hole this exists to close. #[tokio::test] #[ignore] async fn a_denied_sandbox_cannot_reach_the_internet_and_an_open_one_can() { @@ -1159,15 +1173,159 @@ mod live_deny { // the image, the agent or the probe was broken for both. assert!( open_output.contains("HTTP:200"), - "the control must reach the internet, or the deny result means nothing:\n{open_output}" + "the control must reach the internet, or the egressDeny result means \ + nothing:\n{open_output}" ); assert!( !denied_output.contains("HTTP:200"), - "a sandbox under deny reached the internet:\n{denied_output}" + "egressDeny regression: a sandbox under egressDeny reached the internet:\n\ + {denied_output}" ); assert!( denied_output.contains("HTTP:000"), - "deny should fail to connect rather than get some other status:\n{denied_output}" + "egressDeny regression: a denied sandbox should fail to connect rather than get some \ + other status:\n{denied_output}" + ); + } + + /// Reclaims everything a guard run leaves on the probe image: its sessions, then its versions. + /// + /// A MicroVM the run did not terminate bills for hours and no stack delete reaches it, and a + /// surviving version holds the image open so the delete that follows cannot remove it. + #[tokio::test] + #[ignore] + async fn reclaim_the_probe_image() { + let image = std::env::var("PROBE_IMAGE_NAME").expect( + "PROBE_IMAGE_NAME, the image ARN egress-deny-guard-teardown.sh reads from the stack", + ); + let client = client(); + + // Proves the identifier addresses something before the version list is read, so an empty + // list means reclaimed rather than misaddressed: every other caller puts PROBE_IMAGE_NAME + // in a request body, and this is the first to put it in a path. + if let Err(error) = client.get_microvm_image(&image).await { + assert_eq!( + error.code, "REMOTE_RESOURCE_NOT_FOUND", + "the probe image was unreadable for a reason other than being gone, so a reclaim \ + cannot prove anything: {error}" + ); + println!("probe image already reclaimed"); + return; + } + + let mut stranded = Vec::new(); + let mut terminating = Vec::new(); + let mut undeleted = Vec::new(); + for version in client + .list_microvm_image_versions(&image) + .await + .expect("ListMicrovmImageVersions") + { + let version = version.image_version.expect("a version identifier"); + for microvm in client + .list_microvms(&image, &version) + .await + .expect("ListMicrovms") + { + let id = microvm.microvm_id.expect("a MicroVM id"); + // Every MicroVM is attempted before anything fails: one that refuses to + // terminate would otherwise abandon the rest, which bill by the hour. + match client.terminate_microvm(&id).await { + Ok(()) => terminating.push(id), + Err(error) => { + println!("{id}: refused: {error}"); + stranded.push(id); + } + } + } + + // Terminate returns once AWS accepts it, the way suspend documents for itself, so the + // version delete below would race the sessions still holding it. Waiting here turns a + // refusal into a real failure rather than a retry that reads like one. + for id in terminating.drain(..) { + let mut gone = false; + for _ in 0..60 { + match client.get_microvm(&id).await { + Err(error) if error.code == "REMOTE_RESOURCE_NOT_FOUND" => { + gone = true; + break; + } + Ok(current) if current.state.as_deref() == Some("TERMINATED") => { + gone = true; + break; + } + _ => tokio::time::sleep(Duration::from_secs(5)).await, + } + } + if gone { + println!("terminated {id}"); + } else { + println!("{id}: still terminating"); + stranded.push(id); + } + } + + // Best-effort for the same reason termination is: a version that refuses to delete + // would otherwise abandon every version after it, and abort before the report below + // naming what is still billing. + match client + .send::( + Method::DELETE, + &format!("/{API_VERSION}/microvm-images/{image}/versions/{version}"), + &[], + None, + "DeleteMicrovmImageVersion", + ) + .await + { + Ok(_) => println!("deleted version {version}"), + Err(error) => { + println!("version {version}: delete refused: {error}"); + undeleted.push(version); + } + } + } + + assert!( + stranded.is_empty(), + "MicroVMs the reclaim could not terminate keep billing and hold the image open: \ + {stranded:?}" + ); + assert!( + undeleted.is_empty(), + "image versions the reclaim could not delete hold the image open, so the stack \ + delete cannot remove it: {undeleted:?}" + ); + let left = client + .list_microvm_image_versions(&image) + .await + .expect("ListMicrovmImageVersions"); + assert!( + left.is_empty(), + "the probe image still holds versions, so the stack delete cannot remove it: {left:?}" + ); + } + + /// The probe image is gone once its stack is. + /// + /// Asserted rather than assumed: a delete on a `CREATED` image is accepted and removes + /// nothing while its versions survive (`delete_images_versions_first`), so a teardown can + /// report success over an image that is still there. + #[tokio::test] + #[ignore] + async fn the_probe_image_is_gone() { + let image = std::env::var("PROBE_IMAGE_NAME").expect( + "PROBE_IMAGE_NAME, the image ARN egress-deny-guard-teardown.sh reads from the stack", + ); + + let error = client().get_microvm_image(&image).await.expect_err( + "the guard's probe image must not survive its own teardown; if AWS now answers a \ + deleted image with a terminal state instead of a 404, widen this to accept that \ + state rather than dropping the check", + ); + assert_eq!( + error.code, "REMOTE_RESOURCE_NOT_FOUND", + "the probe image is still readable after teardown: {error}" ); } diff --git a/crates/alien-build/examples/sandbox-bundle.rs b/crates/alien-build/examples/sandbox-bundle.rs new file mode 100644 index 000000000..b5f0c4421 --- /dev/null +++ b/crates/alien-build/examples/sandbox-bundle.rs @@ -0,0 +1,32 @@ +//! Writes the bundle a Lambda MicroVM image is built from. +//! +//! ```text +//! cargo run -p alien-build --example sandbox-bundle -- +//! ``` +//! +//! The agent must be an aarch64 Linux binary: MicroVM images accept no other architecture, and +//! an x86 one produces an image that never becomes active. + +use std::path::Path; +use std::process::ExitCode; + +use alien_build::sandbox_bundle::write_bundle; + +fn main() -> ExitCode { + let arguments: Vec = std::env::args().skip(1).collect(); + let [agent, base_image, destination] = arguments.as_slice() else { + eprintln!("usage: sandbox-bundle "); + return ExitCode::FAILURE; + }; + + match write_bundle(Path::new(destination), base_image, Path::new(agent)) { + Ok(()) => { + println!("{destination}"); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("{error}"); + ExitCode::FAILURE + } + } +} diff --git a/scripts/README.md b/scripts/README.md index 2d0937044..565a010aa 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,6 +18,12 @@ Small operational scripts used by root `package.json` commands and GitHub Action - **`write-gke-kubeconfig.sh`** — Used by `configure-e2e-provider-ingress.sh` and GKE Terraform distribution jobs to replace Terraform's static-client kubeconfig with one authenticated as the selected target service account. - **`cleanup-aws-e2e-resources.sh`** — Used by `.github/workflows/e2e-cloud.yml` before and after cloud E2E jobs to remove AWS resources for an E2E slot or explicit resource prefix. Run only with target-account AWS credentials and a scoped `ALIEN_E2E_SLOT` or `ALIEN_E2E_RESOURCE_PREFIX`. +## Sandbox egress guard + +- **`egress-deny-guard-teardown.sh`** — Used by `.github/workflows/egress-deny-guard.yml` before and after the weekly `egressDeny` check to remove the probe stack, and to fail the run if any of it survives. Takes the stack name and needs target-account AWS credentials in both the CLI's and the Rust client's environment (`AWS_TARGET_*`). + + Two preconditions live outside this repository: the OIDC role needs `MaxSessionDuration` of at least 5400 seconds, or every run dies at credential assumption; and a lifecycle rule expiring the artifact bucket's `egress-deny-guard/` prefix is what reclaims a bundle a cancelled run could not delete. + ## Example testing - **`test-examples-local.sh`** (`pnpm test:examples`) — Used by CI Fast and local development to test examples against local source by temporarily injecting `pnpm.overrides`. Always restores `examples/package.json` and `examples/pnpm-lock.yaml` via trap cleanup. diff --git a/scripts/egress-deny-guard-teardown.sh b/scripts/egress-deny-guard-teardown.sh new file mode 100755 index 000000000..a9130e40c --- /dev/null +++ b/scripts/egress-deny-guard-teardown.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# Removes everything the egress-deny guard creates, and fails if any of it survives — except a +# stack it deliberately keeps when a reclaim leaves resources behind, so the next run can find them. +# +# Runs before the guard as well as after it: a stack an earlier run left in a failed state would +# otherwise wedge every run that follows. +# +# Destructive, and assumes no guard run is in flight. The workflow serializes runs through a +# concurrency group; a hand-run invocation has nothing holding that lock. + +set -uo pipefail + +STACK_NAME=${1:?usage: egress-deny-guard-teardown.sh } + +cd "$(dirname "$0")/.." || exit 1 + +run_ignored_test() { + cargo nextest run -p alien-aws-clients --lib --no-tests=fail --run-ignored=all \ + -E "test(=aws::lambda_microvms::live_deny::$1)" +} + +# An absent stack is the success case; anything else — expired credentials, AccessDenied, a +# throttle — must not be read as one. Matching the message is how `cleanup-aws-e2e-resources.sh` +# draws the same distinction. +stack_state() { + local err + if err=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>&1 >/dev/null); then + echo "present" + return 0 + fi + case "$err" in + *"does not exist"*) echo "absent" ;; + *) echo "::error::describe-stacks on $STACK_NAME was inconclusive: $err" >&2; return 1 ;; + esac +} + +state=$(stack_state) || exit 1 +if [ "$state" = "absent" ]; then + echo "no $STACK_NAME stack to remove" + exit 0 +fi + +# A stack that never reached its change set carries no Outputs, which is expected here — that +# state is exactly what this run exists to clean up. A call that fails outright is not: reading it +# as "no outputs" would skip the reclaim and the image check while still exiting 0. +# stderr goes to its own file rather than into the value, so one stray CLI warning cannot make a +# well-formed stack look like a malformed one. +outputs_err=$(mktemp) +trap 'rm -f "$outputs_err"' EXIT +if ! resources=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='DeploymentResources'].OutputValue" \ + --output text 2>"$outputs_err"); then + echo "::error::reading $STACK_NAME outputs was inconclusive: $(cat "$outputs_err")" + exit 1 +fi + +image_arn="" +if [ -n "$resources" ] && [ "$resources" != "None" ]; then + if ! image_arn=$(printf '%s' "$resources" | + jq -re '.[] | select(.type == "sandbox") | .importData.imageArn' 2>/dev/null | head -1); then + echo "::error::$STACK_NAME reported outputs that carry no sandbox image ARN" + exit 1 + fi +fi + +# Outputs are absent in the stack states this script exists to clear, so the image is resolved +# from the stack's own resources instead. Only a stack that genuinely declares no image may leave +# this empty: gating the reclaim and the is-it-gone check on a silently empty value would report a +# clean teardown over an image nobody looked at. +if [ -z "$image_arn" ]; then + if ! image_arn=$(aws cloudformation list-stack-resources --stack-name "$STACK_NAME" \ + --query "StackResourceSummaries[?ResourceType=='AWS::Lambda::MicrovmImage'].PhysicalResourceId" \ + --output text 2>"$outputs_err" | head -1); then + echo "::error::listing $STACK_NAME resources was inconclusive: $(cat "$outputs_err")" + exit 1 + fi + [ "$image_arn" = "None" ] && image_arn="" +fi + +failed=0 + +# Sessions and image versions outlive the stack: the delete below reaches no running MicroVM, and +# a delete on the image is accepted while removing nothing for as long as a version survives. +# So versions go first, and the stack stays standing if that fails: it is the only way back to the +# image, whether through its outputs or its resources, and deleting it would leave MicroVMs +# billing with nothing left pointing at them. +if [ -n "$image_arn" ]; then + echo "::add-mask::$image_arn" + export PROBE_IMAGE_NAME="$image_arn" + if ! run_ignored_test reclaim_the_probe_image; then + echo "::error title=egressDeny guard::the reclaim did not clear the probe image, so \ +$STACK_NAME stays up — it is what the next run needs to reach whatever is left. The failure \ +above says whether resources survived or the reclaim could not run." + exit 1 + fi +fi + +aws cloudformation delete-stack --stack-name "$STACK_NAME" || failed=1 +aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || failed=1 + +# Runs whatever the delete reported: a stack can reach DELETE_COMPLETE over an image the delete +# was accepted for and never removed, and this is the only thing that would notice. +if [ -n "$image_arn" ]; then + run_ignored_test the_probe_image_is_gone || failed=1 +fi + +state=$(stack_state) || exit 1 +if [ "$state" = "present" ]; then + echo "::error title=egressDeny guard::$STACK_NAME survived its own teardown" + exit 1 +fi + +exit "$failed" From acac9f4d1e149c27fd9538ded991506431af9564 Mon Sep 17 00:00:00 2001 From: Itamar Zand <133867530+ItamarZand88@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:57:28 +0300 Subject: [PATCH 2/2] feat(sandbox): source the bundled agent from a published image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundle zip embedded the agent binary, so shipping an agent meant producing and re-publishing a per-region artifact nothing builds today. The Dockerfile can copy the agent out of a published container image instead — the image build already pulls a public base image, so this adds no reachability it did not have. AgentSource keeps both shapes: Image is the shipping path and carries only a Dockerfile in the zip; Binary embeds a local build, which the egressDeny guard keeps using because it exists to test this commit's agent rather than a published one. --- .github/workflows/egress-deny-guard.yml | 4 +- crates/alien-build/examples/sandbox-bundle.rs | 32 ++- crates/alien-build/src/sandbox_bundle.rs | 193 ++++++++++++++---- 3 files changed, 182 insertions(+), 47 deletions(-) diff --git a/.github/workflows/egress-deny-guard.yml b/.github/workflows/egress-deny-guard.yml index 8e834046b..af384c27c 100644 --- a/.github/workflows/egress-deny-guard.yml +++ b/.github/workflows/egress-deny-guard.yml @@ -76,8 +76,10 @@ jobs: run: | set -euo pipefail mkdir -p .guard + # --agent-binary on purpose: the guard exists to test THIS commit's agent, so it + # embeds the fresh build rather than pulling a published image. cargo run -p alien-build --example sandbox-bundle -- \ - "$AGENT_BINARY" "$SANDBOX_BASE_IMAGE" .guard/sandbox-bundle.zip + --agent-binary "$AGENT_BINARY" "$SANDBOX_BASE_IMAGE" .guard/sandbox-bundle.zip # Per-run key, so nothing collides. A cancelled run cannot remove its own object and # no later run can guess the key: expiry on the `egress-deny-guard/` prefix is what # reclaims it, and that lives in the bucket's lifecycle configuration. diff --git a/crates/alien-build/examples/sandbox-bundle.rs b/crates/alien-build/examples/sandbox-bundle.rs index b5f0c4421..c8b0cd104 100644 --- a/crates/alien-build/examples/sandbox-bundle.rs +++ b/crates/alien-build/examples/sandbox-bundle.rs @@ -1,25 +1,39 @@ //! Writes the bundle a Lambda MicroVM image is built from. //! //! ```text -//! cargo run -p alien-build --example sandbox-bundle -- +//! cargo run -p alien-build --example sandbox-bundle -- --agent-binary +//! cargo run -p alien-build --example sandbox-bundle -- --agent-image //! ``` //! -//! The agent must be an aarch64 Linux binary: MicroVM images accept no other architecture, and -//! an x86 one produces an image that never becomes active. +//! `--agent-image` is the shipping path: the bundle carries only a Dockerfile that copies the +//! agent out of the published image. `--agent-binary` embeds a local build for CI/dev instead, +//! and must be aarch64 Linux — MicroVM images accept no other architecture. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::ExitCode; -use alien_build::sandbox_bundle::write_bundle; +use alien_build::sandbox_bundle::{write_bundle, AgentSource}; fn main() -> ExitCode { let arguments: Vec = std::env::args().skip(1).collect(); - let [agent, base_image, destination] = arguments.as_slice() else { - eprintln!("usage: sandbox-bundle "); - return ExitCode::FAILURE; + let usage = || { + eprintln!( + "usage: sandbox-bundle --agent-binary |--agent-image \ + " + ); + ExitCode::FAILURE }; - match write_bundle(Path::new(destination), base_image, Path::new(agent)) { + let [mode, agent, base_image, destination] = arguments.as_slice() else { + return usage(); + }; + let agent = match mode.as_str() { + "--agent-binary" => AgentSource::Binary(PathBuf::from(agent)), + "--agent-image" => AgentSource::Image(agent.clone()), + _ => return usage(), + }; + + match write_bundle(Path::new(destination), base_image, &agent) { Ok(()) => { println!("{destination}"); ExitCode::SUCCESS diff --git a/crates/alien-build/src/sandbox_bundle.rs b/crates/alien-build/src/sandbox_bundle.rs index 13f5f1873..148ea1cda 100644 --- a/crates/alien-build/src/sandbox_bundle.rs +++ b/crates/alien-build/src/sandbox_bundle.rs @@ -34,30 +34,58 @@ pub use alien_core::sandbox_process::AGENT_PORT; /// Name the agent binary must have inside the bundle. pub const AGENT_FILENAME: &str = "alien-sandbox-agent"; -/// Renders the Dockerfile for a sandbox image built on `base_image`. -/// -/// The agent runs as root so it can drop to [`EXEC_UID`] before every spawn; inside a MicroVM -/// that is contained by hardware virtualisation, which is the tenant boundary. A shared-kernel -/// backend must give the agent `CAP_SETUID` instead of root. -pub fn dockerfile(base_image: &str) -> Result { - // Checked here rather than by the callers: this is the one place the value crosses into - // generated content, and a reference carrying a newline writes its own Dockerfile directives. - if base_image.is_empty() - || base_image +/// Where the agent comes from at image-build time: `Image` copies it out of a published +/// container image (no binary in the bundle, a new agent is a tag change); `Binary` embeds a +/// local build instead, for CI on the current commit and for pre-publish development. +#[derive(Debug, Clone)] +pub enum AgentSource { + /// A published image holding the agent at [`AGENT_PATH`]. + Image(String), + /// A local agent binary, zipped into the bundle beside the Dockerfile. + Binary(std::path::PathBuf), +} + +/// Refuses a reference that cannot cross into generated content: a newline in it writes its own +/// Dockerfile directives. +fn checked_reference<'a>(reference: &'a str, what: &str) -> Result<&'a str> { + if reference.is_empty() + || reference .chars() .any(|c| c.is_whitespace() || c.is_control()) { return Err(AlienError::new(ErrorData::BuildConfigInvalid { - message: format!("base image reference '{base_image}' is not a valid image reference"), + message: format!( + "{what} reference '{reference}' is empty or carries whitespace or control \ + characters, which cannot cross into a generated Dockerfile" + ), })); } + Ok(reference) +} + +/// Renders the Dockerfile for a sandbox image built on `base_image`. +/// +/// The agent runs as root so it can drop to [`EXEC_UID`] before every spawn; inside a MicroVM +/// that is contained by hardware virtualisation, which is the tenant boundary. A shared-kernel +/// backend must give the agent `CAP_SETUID` instead of root. +pub fn dockerfile(base_image: &str, agent: &AgentSource) -> Result { + let base_image = checked_reference(base_image, "base image")?; + // Both lines pin ownership and mode themselves: the untrusted code the agent supervises must + // not be able to rewrite the supervisor, whichever way the agent arrived. + let copy_agent = match agent { + AgentSource::Image(image) => { + let image = checked_reference(image, "agent image")?; + format!("COPY --from={image} --chown=0:0 --chmod=0755 {AGENT_PATH} {AGENT_PATH}") + } + AgentSource::Binary(_) => { + format!("COPY --chown=0:0 --chmod=0755 {AGENT_FILENAME} {AGENT_PATH}") + } + }; Ok(format!( r#"FROM {base_image} -# Root-owned and not writable by the exec uid: the untrusted code the agent supervises must not -# be able to rewrite the supervisor. -COPY --chown=0:0 --chmod=0755 {AGENT_FILENAME} {AGENT_PATH} +{copy_agent} # Written with numeric ids and a plain append rather than useradd/adduser, which differ across # base distributions. Linux runs a process under a uid with no passwd entry, but some tooling @@ -84,35 +112,46 @@ ENTRYPOINT ["{AGENT_PATH}"] )) } -/// Writes the bundle AWS builds a MicroVM image from: the rendered Dockerfile and the agent -/// binary beside it, zipped. +/// Writes the bundle AWS builds a MicroVM image from: the rendered Dockerfile, plus the agent +/// binary beside it when the agent is a local build rather than a published image. /// /// The archive is flat on purpose — `CreateMicrovmImage` looks for the Dockerfile at the root, /// and a nested directory produces a build failure minutes in rather than a rejected request. -pub fn write_bundle(destination: &Path, base_image: &str, agent_binary: &Path) -> Result<()> { +pub fn write_bundle(destination: &Path, base_image: &str, agent: &AgentSource) -> Result<()> { let failed = |operation: &str, path: &Path| ErrorData::FileOperationFailed { operation: operation.to_string(), file_path: path.display().to_string(), reason: "could not assemble the sandbox image bundle".to_string(), }; - let agent = std::fs::read(agent_binary) - .into_alien_error() - .context(failed("read", agent_binary))?; + // Every fallible input resolves before the archive exists, so no failure — a bad reference + // or an unreadable agent binary — leaves a truncated zip behind. + let dockerfile = dockerfile(base_image, agent)?; + let agent_bytes = match agent { + AgentSource::Binary(agent_binary) => Some( + std::fs::read(agent_binary) + .into_alien_error() + .context(failed("read", agent_binary))?, + ), + AgentSource::Image(_) => None, + }; + let archive = File::create(destination) .into_alien_error() .context(failed("create", destination))?; let mut zip = ZipWriter::new(archive); - // 0755 on the agent so the entry is already executable; the Dockerfile's `--chmod` covers - // builders that drop archive modes, and neither alone is reliable across both. - let options: SimpleFileOptions = SimpleFileOptions::default().unix_permissions(0o755); - zip.start_file(AGENT_FILENAME, options) - .into_alien_error() - .context(failed("write", destination))?; - zip.write_all(&agent) - .into_alien_error() - .context(failed("write", destination))?; + if let Some(bytes) = agent_bytes { + // 0755 on the agent so the entry is already executable; the Dockerfile's `--chmod` covers + // builders that drop archive modes, and neither alone is reliable across both. + let options: SimpleFileOptions = SimpleFileOptions::default().unix_permissions(0o755); + zip.start_file(AGENT_FILENAME, options) + .into_alien_error() + .context(failed("write", destination))?; + zip.write_all(&bytes) + .into_alien_error() + .context(failed("write", destination))?; + } zip.start_file( "Dockerfile", @@ -120,7 +159,7 @@ pub fn write_bundle(destination: &Path, base_image: &str, agent_binary: &Path) - ) .into_alien_error() .context(failed("write", destination))?; - zip.write_all(dockerfile(base_image)?.as_bytes()) + zip.write_all(dockerfile.as_bytes()) .into_alien_error() .context(failed("write", destination))?; @@ -142,14 +181,27 @@ mod tests { "", "alpine\tlatest", ] { - super::dockerfile(reference) - .expect_err(&format!("{reference:?} must not render into a Dockerfile")); + super::dockerfile( + reference, + &super::AgentSource::Image("agent:v1".to_string()), + ) + .expect_err(&format!("{reference:?} must not render into a Dockerfile")); + super::dockerfile( + "ubuntu:24.04", + &super::AgentSource::Image(reference.to_string()), + ) + .expect_err(&format!( + "{reference:?} must not render as an agent image either" + )); } // The control arm: an ordinary reference still renders, so the guard is not refusing // everything. - let rendered = super::dockerfile("public.ecr.aws/lambda/microvms:al2023-minimal") - .expect("an ordinary reference renders"); + let rendered = super::dockerfile( + "public.ecr.aws/lambda/microvms:al2023-minimal", + &super::AgentSource::Image("agent:v1".to_string()), + ) + .expect("an ordinary reference renders"); assert!(rendered.starts_with("FROM public.ecr.aws/lambda/microvms:al2023-minimal")); } @@ -157,8 +209,16 @@ mod tests { /// The properties below are the image's half of the supervisor boundary. A base image is /// caller-supplied, so these assertions are about what Alien adds on top of it. + fn embedded_agent() -> AgentSource { + AgentSource::Binary(std::path::PathBuf::from("unused-in-render")) + } + fn rendered() -> String { - dockerfile("public.ecr.aws/lambda/microvms:al2023-minimal").expect("a valid reference") + dockerfile( + "public.ecr.aws/lambda/microvms:al2023-minimal", + &embedded_agent(), + ) + .expect("a valid reference") } #[test] @@ -230,7 +290,8 @@ mod tests { std::fs::write(&agent, b"\x7fELF-not-really").expect("agent"); let bundle = dir.path().join("sandbox.zip"); - write_bundle(&bundle, "ubuntu:24.04", &agent).expect("writes the bundle"); + write_bundle(&bundle, "ubuntu:24.04", &AgentSource::Binary(agent)) + .expect("writes the bundle"); let file = std::fs::File::open(&bundle).expect("opens"); let mut archive = zip::ZipArchive::new(file).expect("reads as a zip"); @@ -254,6 +315,64 @@ mod tests { assert!(contents.starts_with("FROM ubuntu:24.04")); } + /// Image-sourced bundles carry no agent binary: the zip holds only the Dockerfile. + #[test] + fn an_image_sourced_bundle_carries_only_the_dockerfile() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let bundle = dir.path().join("sandbox.zip"); + + write_bundle( + &bundle, + "ubuntu:24.04", + &AgentSource::Image("public.ecr.aws/acme/sandbox-agent:v1".to_string()), + ) + .expect("writes the bundle"); + + let file = std::fs::File::open(&bundle).expect("opens"); + let mut archive = zip::ZipArchive::new(file).expect("zip"); + assert_eq!(archive.len(), 1, "nothing but the Dockerfile belongs here"); + + let mut dockerfile = String::new(); + std::io::Read::read_to_string( + &mut archive.by_name("Dockerfile").expect("dockerfile entry"), + &mut dockerfile, + ) + .expect("reads"); + assert!( + dockerfile.contains(&format!( + "COPY --from=public.ecr.aws/acme/sandbox-agent:v1 \ + --chown=0:0 --chmod=0755 {AGENT_PATH} {AGENT_PATH}" + )) || dockerfile.contains(&format!( + "COPY --from=public.ecr.aws/acme/sandbox-agent:v1 --chown=0:0 --chmod=0755 {AGENT_PATH} {AGENT_PATH}" + )), + "the agent must be copied out of the named image, root-owned and 0755:\n{dockerfile}" + ); + assert!( + !dockerfile.contains(&format!("COPY --chown=0:0 --chmod=0755 {AGENT_FILENAME} ")), + "the local-file COPY must not appear when the agent comes from an image" + ); + } + + /// A failed input must leave nothing at the destination — a truncated zip uploaded by a + /// caller that only checked the exit path fails ~160s into an image build instead of here. + #[test] + fn a_missing_agent_binary_leaves_no_bundle_behind() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let bundle = dir.path().join("sandbox.zip"); + + write_bundle( + &bundle, + "ubuntu:24.04", + &AgentSource::Binary(dir.path().join("does-not-exist")), + ) + .expect_err("an unreadable agent must fail the bundle"); + + assert!( + !bundle.exists(), + "no partial archive may be left at the destination" + ); + } + /// The agent entry must survive as an executable. A builder that honours archive modes and /// one that does not both have to produce a runnable binary, which is why the Dockerfile /// also carries `--chmod`. @@ -263,7 +382,7 @@ mod tests { let agent = dir.path().join("agent-bin"); std::fs::write(&agent, b"binary").expect("agent"); let bundle = dir.path().join("sandbox.zip"); - write_bundle(&bundle, "ubuntu:24.04", &agent).expect("writes"); + write_bundle(&bundle, "ubuntu:24.04", &AgentSource::Binary(agent)).expect("writes"); let file = std::fs::File::open(&bundle).expect("opens"); let mut archive = zip::ZipArchive::new(file).expect("zip");