From 9f14cbdb5b8891b3cb5f0beae7372db08e21007f Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 21:27:19 +0200 Subject: [PATCH 01/35] feat: deploy rybbit for rubykatzen.com through flightdeck itself First real (non-CI-test) self-consumption of deploy-shared.yml: flightdeck deploys its own rybbit instance at rybbit.rubykatzen.com, via an isolated rubykatzen-com user on the existing hawkeye host (Tailscale IP 100.75.50.2), separate from dupmachine's own services on that box (own traefik on 8080/8443 to avoid the port conflict, own age keypair, own env manifest). - projects/rubykatzen/rubykatzen-com.yml: publish-sops-env manifest (APPS=traefik,rybbit, Cloudflare DNS challenge cert resolver). - keys/rubykatzen-com.pub: age recipient for this deployment's encrypted env. Private key generated locally, not committed -- goes on the server at flightdeck_sops_age_key_file, out of band. - publish-rubykatzen-com-env.yml: renders + encrypts the manifest, publishes it to the rubykatzen-com-config release tag. - deploy-rubykatzen-com.yml: calls deploy-shared.yml@v0.3.0 (pinned, same as a real external consumer would) against 100.75.50.2. Blocked on, not part of this diff: DEPLOY_SSH_PRIVATE_KEY secret, TAILSCALE_OAUTH_CLIENT_ID/SECRET, RUBYKATZEN_COM_CLOUDFLARE_TOKEN secret, and server-side prep (create rubykatzen-com user, install the age private key, cloudflared tunnel). workflow_dispatch-only until those land. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy-rubykatzen-com.yml | 19 ++++++++++++ .../workflows/publish-rubykatzen-com-env.yml | 31 +++++++++++++++++++ keys/rubykatzen-com.pub | 1 + projects/rubykatzen/rubykatzen-com.yml | 17 ++++++++++ 4 files changed, 68 insertions(+) create mode 100644 .github/workflows/deploy-rubykatzen-com.yml create mode 100644 .github/workflows/publish-rubykatzen-com-env.yml create mode 100644 keys/rubykatzen-com.pub create mode 100644 projects/rubykatzen/rubykatzen-com.yml diff --git a/.github/workflows/deploy-rubykatzen-com.yml b/.github/workflows/deploy-rubykatzen-com.yml new file mode 100644 index 0000000..6e028df --- /dev/null +++ b/.github/workflows/deploy-rubykatzen-com.yml @@ -0,0 +1,19 @@ +name: Deploy rubykatzen.com +on: + workflow_dispatch: +jobs: + deploy: + uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v0.3.0 + with: + inventory: "100.75.50.2" + user: rubykatzen-com + extra-vars: | + {"flightdeck_env_ref":"${{ github.repository }}@rubykatzen-com-config:rubykatzen-com.sops.env", + "flightdeck_extra_refs":[], + "flightdeck_path":"~/flightdeck", + "flightdeck_keep_releases":5, + "flightdeck_sops_age_key_file":"/home/rubykatzen-com/.config/sops/age/keys.txt"} + tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} + secrets: + ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} diff --git a/.github/workflows/publish-rubykatzen-com-env.yml b/.github/workflows/publish-rubykatzen-com-env.yml new file mode 100644 index 0000000..6dcb66d --- /dev/null +++ b/.github/workflows/publish-rubykatzen-com-env.yml @@ -0,0 +1,31 @@ +name: Publish rubykatzen.com env +on: + workflow_dispatch: + push: + branches: [main] + paths: + - projects/rubykatzen/rubykatzen-com.yml + - keys/rubykatzen-com.pub +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + - name: Ensure release exists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release view rubykatzen-com-config --repo "${{ github.repository }}" >/dev/null 2>&1 || \ + gh release create rubykatzen-com-config --repo "${{ github.repository }}" \ + --title "rubykatzen.com config" \ + --notes "Encrypted env for the rubykatzen.com self-hosted deployment." \ + --target main + - uses: ./.github/actions/publish-sops-env + with: + manifest: projects/rubykatzen/rubykatzen-com.yml + token: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} + GITHUB_VARS_JSON: ${{ toJson(vars) }} diff --git a/keys/rubykatzen-com.pub b/keys/rubykatzen-com.pub new file mode 100644 index 0000000..106f30c --- /dev/null +++ b/keys/rubykatzen-com.pub @@ -0,0 +1 @@ +age1tc5rvv3h80w6er4888rnah8w68nvz2lwqr2fg32s5hu0u2pfcdrqhradth diff --git a/projects/rubykatzen/rubykatzen-com.yml b/projects/rubykatzen/rubykatzen-com.yml new file mode 100644 index 0000000..9d25024 --- /dev/null +++ b/projects/rubykatzen/rubykatzen-com.yml @@ -0,0 +1,17 @@ +release_tag: rubykatzen-com-config +release_asset: rubykatzen-com.sops.env + +keys: + - rubykatzen-com + +env: + APPS: RUBYKATZEN_COM_APPS + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN + APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL + APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER + APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN + APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD + APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 + APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE + TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT + TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT From 7cbcf659b9b23af2a8aed275363329c29bbb3ddc Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 21:36:51 +0200 Subject: [PATCH 02/35] refactor: rename projects/rubykatzen/rubykatzen-com.yml to targets/hawkeye.yml The manifest identity is the server + isolated user + age key it decrypts on (hawkeye), not the domain/service currently running there (rubykatzen.com/rybbit) or a repo-grouping level that doesn't apply to self-hosting flightdeck's own deploys. Matches the existing convention elsewhere of naming manifests/assets after the server (mainframe, hawkeye), and drops the redundant projects/rubykatzen/ nesting -- one flat targets/ directory, one file per deploy target. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/publish-rubykatzen-com-env.yml | 4 ++-- projects/rubykatzen/rubykatzen-com.yml => targets/hawkeye.yml | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename projects/rubykatzen/rubykatzen-com.yml => targets/hawkeye.yml (100%) diff --git a/.github/workflows/publish-rubykatzen-com-env.yml b/.github/workflows/publish-rubykatzen-com-env.yml index 6dcb66d..77a00c4 100644 --- a/.github/workflows/publish-rubykatzen-com-env.yml +++ b/.github/workflows/publish-rubykatzen-com-env.yml @@ -4,7 +4,7 @@ on: push: branches: [main] paths: - - projects/rubykatzen/rubykatzen-com.yml + - targets/hawkeye.yml - keys/rubykatzen-com.pub jobs: publish: @@ -24,7 +24,7 @@ jobs: --target main - uses: ./.github/actions/publish-sops-env with: - manifest: projects/rubykatzen/rubykatzen-com.yml + manifest: targets/hawkeye.yml token: ${{ secrets.GITHUB_TOKEN }} env: GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} diff --git a/projects/rubykatzen/rubykatzen-com.yml b/targets/hawkeye.yml similarity index 100% rename from projects/rubykatzen/rubykatzen-com.yml rename to targets/hawkeye.yml From 036598794edf92fc8780aedbdf2eda510e1343e2 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 21:41:19 +0200 Subject: [PATCH 03/35] refactor: name key, release tag, and asset after the target (hawkeye) Same reasoning as the targets/ rename: the age key, release tag, and asset identify the deploy target (server + isolated user + key), not the service currently running there. Renamed rubykatzen-com.pub -> hawkeye.pub, rubykatzen-com-config -> hawkeye-config, rubykatzen-com.sops.env -> hawkeye.sops.env. The RUBYKATZEN_COM_* GitHub variable/secret names are untouched -- those describe the service's own config content, not the target identity, and the rubykatzen-com OS username/home path on the server is a separate, correctly-still-rubykatzen-com concept. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy-rubykatzen-com.yml | 2 +- .github/workflows/publish-rubykatzen-com-env.yml | 10 +++++----- keys/{rubykatzen-com.pub => hawkeye.pub} | 0 targets/hawkeye.yml | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) rename keys/{rubykatzen-com.pub => hawkeye.pub} (100%) diff --git a/.github/workflows/deploy-rubykatzen-com.yml b/.github/workflows/deploy-rubykatzen-com.yml index 6e028df..863cff6 100644 --- a/.github/workflows/deploy-rubykatzen-com.yml +++ b/.github/workflows/deploy-rubykatzen-com.yml @@ -8,7 +8,7 @@ jobs: inventory: "100.75.50.2" user: rubykatzen-com extra-vars: | - {"flightdeck_env_ref":"${{ github.repository }}@rubykatzen-com-config:rubykatzen-com.sops.env", + {"flightdeck_env_ref":"${{ github.repository }}@hawkeye-config:hawkeye.sops.env", "flightdeck_extra_refs":[], "flightdeck_path":"~/flightdeck", "flightdeck_keep_releases":5, diff --git a/.github/workflows/publish-rubykatzen-com-env.yml b/.github/workflows/publish-rubykatzen-com-env.yml index 77a00c4..0809249 100644 --- a/.github/workflows/publish-rubykatzen-com-env.yml +++ b/.github/workflows/publish-rubykatzen-com-env.yml @@ -5,7 +5,7 @@ on: branches: [main] paths: - targets/hawkeye.yml - - keys/rubykatzen-com.pub + - keys/hawkeye.pub jobs: publish: runs-on: ubuntu-latest @@ -17,10 +17,10 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release view rubykatzen-com-config --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create rubykatzen-com-config --repo "${{ github.repository }}" \ - --title "rubykatzen.com config" \ - --notes "Encrypted env for the rubykatzen.com self-hosted deployment." \ + gh release view hawkeye-config --repo "${{ github.repository }}" >/dev/null 2>&1 || \ + gh release create hawkeye-config --repo "${{ github.repository }}" \ + --title "hawkeye config" \ + --notes "Encrypted env for the hawkeye deploy target." \ --target main - uses: ./.github/actions/publish-sops-env with: diff --git a/keys/rubykatzen-com.pub b/keys/hawkeye.pub similarity index 100% rename from keys/rubykatzen-com.pub rename to keys/hawkeye.pub diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 9d25024..688d2b1 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,8 +1,8 @@ -release_tag: rubykatzen-com-config -release_asset: rubykatzen-com.sops.env +release_tag: hawkeye-config +release_asset: hawkeye.sops.env keys: - - rubykatzen-com + - hawkeye env: APPS: RUBYKATZEN_COM_APPS From db021de1e2cd66ce7818d6d1971b722b07543b33 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 21:46:25 +0200 Subject: [PATCH 04/35] feat: add publish-sops-env-shared.yml, use it as our own consumer We were the odd one out: deploy-rubykatzen-com.yml already calls deploy-shared.yml as a real interface, but publish-rubykatzen-com-env.yml hand-assembled its own "ensure release exists" step around the local publish-sops-env composite action instead of going through an equivalent reusable workflow -- because that reusable workflow didn't exist yet, only the composite action did. Add publish-sops-env-shared.yml (checkout, resolve release tag -- input or read from the manifest's own release_tag, ensure the release exists, call publish-sops-env) and rewrite our own publish-rubykatzen-com-env.yml to just call it, matching the same consumer interface pattern deploy-rubykatzen-com.yml already uses. Note: this one needs `secrets: inherit` on the caller, unlike deploy-shared.yml's explicit named secrets -- the manifest can reference an arbitrary, per-target set of secret/variable names, so there's no fixed schema to declare individually. Pinned to @main for now since this file doesn't exist in any released tag yet; re-pin to a real version once the next release ships, same as deploy-rubykatzen-com.yml does for deploy-shared.yml@v0.3.0. Co-Authored-By: Claude Sonnet 5 --- .../workflows/publish-rubykatzen-com-env.yml | 25 ++------- .github/workflows/publish-sops-env-shared.yml | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/publish-sops-env-shared.yml diff --git a/.github/workflows/publish-rubykatzen-com-env.yml b/.github/workflows/publish-rubykatzen-com-env.yml index 0809249..b4d7e2d 100644 --- a/.github/workflows/publish-rubykatzen-com-env.yml +++ b/.github/workflows/publish-rubykatzen-com-env.yml @@ -8,24 +8,7 @@ on: - keys/hawkeye.pub jobs: publish: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - - name: Ensure release exists - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release view hawkeye-config --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create hawkeye-config --repo "${{ github.repository }}" \ - --title "hawkeye config" \ - --notes "Encrypted env for the hawkeye deploy target." \ - --target main - - uses: ./.github/actions/publish-sops-env - with: - manifest: targets/hawkeye.yml - token: ${{ secrets.GITHUB_TOKEN }} - env: - GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} - GITHUB_VARS_JSON: ${{ toJson(vars) }} + uses: rubykatzen/flightdeck/.github/workflows/publish-sops-env-shared.yml@main + with: + manifest: targets/hawkeye.yml + secrets: inherit diff --git a/.github/workflows/publish-sops-env-shared.yml b/.github/workflows/publish-sops-env-shared.yml new file mode 100644 index 0000000..29dfef1 --- /dev/null +++ b/.github/workflows/publish-sops-env-shared.yml @@ -0,0 +1,54 @@ +name: Publish SOPS env (shared) +on: + workflow_call: + inputs: + manifest: + description: Path to the YAML env manifest. + type: string + required: true + release-tag: + description: Release tag to ensure exists and publish the encrypted asset to. Defaults to the manifest's own release_tag. + type: string + default: "" +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + - name: Resolve release tag + id: tag + shell: bash + env: + INPUT_TAG: ${{ inputs.release-tag }} + MANIFEST: ${{ inputs.manifest }} + run: | + tag="$INPUT_TAG" + if [ -z "$tag" ]; then + tag="$(grep -m1 '^release_tag:' "$MANIFEST" | sed 's/^release_tag:[[:space:]]*//')" + fi + if [ -z "$tag" ]; then + echo "::error::No release-tag input and no release_tag in $MANIFEST" >&2 + exit 1 + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + - name: Ensure release exists + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.tag.outputs.tag }} + run: | + gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1 || \ + gh release create "$TAG" --repo "${{ github.repository }}" \ + --title "$TAG" \ + --notes "Encrypted env release." \ + --target main + - uses: ./.github/actions/publish-sops-env + with: + manifest: ${{ inputs.manifest }} + release-tag: ${{ steps.tag.outputs.tag }} + token: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} + GITHUB_VARS_JSON: ${{ toJson(vars) }} From 20d97680f41524ddbb798a0bc4d5f0f0051ea775 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 21:59:04 +0200 Subject: [PATCH 05/35] refactor!: replace deploy-shared.yml's raw extra-vars JSON with a clean interface The old interface leaked Ansible directly to callers -- extra-vars was a hand-written JSON blob using flightdeck_* variable names lifted straight from ansible/deploy.yml. Callers shouldn't need to know this runs on Ansible at all. Replace with named inputs in plain deploy vocabulary: app-ref, env-ref, extra-refs (comma-separated), path, keep-releases, sops-age-key-file. The workflow builds the ansible -e JSON internally via jq. No behavior change for existing values, just the surface. Added defaults for the ones that have a sensible one: - path: ~/flightdeck (matches every example/consumer so far) - keep-releases: 5 - sops-age-key-file: ~/.config/sops/age/keys.txt for the connecting `user`, computed rather than hardcoded since it depends on which user connects -- can't be a static workflow_call default. We have no external consumers of this interface yet to break, so doing this now rather than carrying the leaky version forward. Also updates our own deploy-rubykatzen-com.yml to the new interface (drops explicit path/keep-releases/sops-age-key-file entirely since the new defaults already match what we were passing), and README's deploy-shared.yml documentation. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy-rubykatzen-com.yml | 9 +-- .github/workflows/deploy-shared.yml | 71 ++++++++++++++++++--- README.md | 18 +++--- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/.github/workflows/deploy-rubykatzen-com.yml b/.github/workflows/deploy-rubykatzen-com.yml index 863cff6..9f58173 100644 --- a/.github/workflows/deploy-rubykatzen-com.yml +++ b/.github/workflows/deploy-rubykatzen-com.yml @@ -3,16 +3,11 @@ on: workflow_dispatch: jobs: deploy: - uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v0.3.0 + uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@main with: inventory: "100.75.50.2" user: rubykatzen-com - extra-vars: | - {"flightdeck_env_ref":"${{ github.repository }}@hawkeye-config:hawkeye.sops.env", - "flightdeck_extra_refs":[], - "flightdeck_path":"~/flightdeck", - "flightdeck_keep_releases":5, - "flightdeck_sops_age_key_file":"/home/rubykatzen-com/.config/sops/age/keys.txt"} + env-ref: "${{ github.repository }}@hawkeye-config:hawkeye.sops.env" tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} secrets: ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 73ba170..ea5792e 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -10,10 +10,30 @@ on: description: SSH user for the Ansible connection. type: string default: root - extra-vars: - description: "JSON object passed to ansible-playbook as -e (flightdeck_env_ref, flightdeck_path, etc). flightdeck_app_ref defaults to the ref this workflow was called at (the @tag on the uses: line) and only needs to be set here to override that." + app-ref: + description: "Flightdeck ref to deploy. Defaults to the ref this workflow was called at (the @tag on the uses: line)." + type: string + default: "" + env-ref: + description: Release ref for the encrypted env package, in owner/repo@tag:asset format. type: string required: true + extra-refs: + description: Comma-separated release refs for optional extra app bundles. + type: string + default: "" + path: + description: Base path on the target host for releases, shared files, and the current symlink. + type: string + default: "~/flightdeck" + keep-releases: + description: Number of past releases to keep on the target host. + type: number + default: 5 + sops-age-key-file: + description: Path to the server-local SOPS age key file. Defaults to ~/.config/sops/age/keys.txt for the connecting user. + type: string + default: "" tailscale-oauth-client-id: description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the inventory hosts). type: string @@ -33,14 +53,19 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Resolve flightdeck ref + - name: Resolve app ref id: ref shell: bash + env: + APP_REF_INPUT: ${{ inputs.app-ref }} run: | - ref="${{ github.workflow_ref }}" - ref="${ref#*@}" - ref="${ref#refs/tags/}" - ref="${ref#refs/heads/}" + ref="$APP_REF_INPUT" + if [ -z "$ref" ]; then + ref="${{ github.workflow_ref }}" + ref="${ref#*@}" + ref="${ref#refs/tags/}" + ref="${ref#refs/heads/}" + fi echo "ref=$ref" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v7 with: @@ -62,6 +87,35 @@ jobs: echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" ssh-add - <<< "${{ secrets.ssh-private-key }}" + - name: Build extra-vars + id: vars + shell: bash + env: + APP_REF: rubykatzen/flightdeck@${{ steps.ref.outputs.ref }} + ENV_REF: ${{ inputs.env-ref }} + EXTRA_REFS: ${{ inputs.extra-refs }} + DEPLOY_PATH: ${{ inputs.path }} + KEEP_RELEASES: ${{ inputs.keep-releases }} + SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} + SSH_USER: ${{ inputs.user }} + run: | + sops_key_file="$SOPS_KEY_FILE" + if [ -z "$sops_key_file" ]; then + sops_key_file="/home/$SSH_USER/.config/sops/age/keys.txt" + fi + extra_refs_json="[]" + if [ -n "$EXTRA_REFS" ]; then + extra_refs_json="$(printf '%s' "$EXTRA_REFS" | jq -R 'split(",")')" + fi + json="$(jq -n \ + --arg app_ref "$APP_REF" \ + --arg env_ref "$ENV_REF" \ + --argjson extra_refs "$extra_refs_json" \ + --arg path "$DEPLOY_PATH" \ + --argjson keep_releases "$KEEP_RELEASES" \ + --arg sops_key_file "$sops_key_file" \ + '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_extra_refs: $extra_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" + echo "json=$json" >> "$GITHUB_OUTPUT" - name: Run playbook shell: bash env: @@ -70,5 +124,4 @@ jobs: ansible-playbook ansible/deploy.yml \ -i "${{ inputs.inventory }}" \ -u "${{ inputs.user }}" \ - -e "{\"flightdeck_app_ref\":\"rubykatzen/flightdeck@${{ steps.ref.outputs.ref }}\"}" \ - -e "${{ inputs.extra-vars }}" + -e "${{ steps.vars.outputs.json }}" diff --git a/README.md b/README.md index 6ec9638..e2a2b07 100644 --- a/README.md +++ b/README.md @@ -519,7 +519,9 @@ Secrets take precedence over Variables when both contain the same source key. Ev ### `deploy-shared.yml` -Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied inventory. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `flightdeck_env_ref` typically references that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. +Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied inventory. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `env-ref` typically references that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. + +The interface is plain deploy vocabulary, not Ansible's — callers never see `flightdeck_*` variable names or hand-write `-e` JSON; the workflow builds that internally. Tailscale is optional, not a dependency of this workflow: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the inventory hosts, or reaches them some other way. @@ -530,12 +532,12 @@ jobs: with: inventory: 100.64.0.1,100.64.0.2 # required user: root # default: root - extra-vars: | - {"flightdeck_env_ref":"${{ github.repository }}@latest:.sops.env", - "flightdeck_extra_refs":[], - "flightdeck_path":"~/flightdeck", - "flightdeck_keep_releases":5, - "flightdeck_sops_age_key_file":"/home/deploy/.config/sops/age/keys.txt"} + env-ref: "${{ github.repository }}@latest:.sops.env" # required + # app-ref: v1.2.3 # optional, default: the @tag on the uses: line above + # extra-refs: owner/repo@latest # optional, comma-separated, default: none + # path: ~/flightdeck # optional, default shown + # keep-releases: 5 # optional, default shown + # sops-age-key-file: /home/deploy/.config/sops/age/keys.txt # optional, default: ~/.config/sops/age/keys.txt for `user` tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) tailscale-tags: tag:ci # default: tag:ci secrets: @@ -543,7 +545,7 @@ jobs: tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # optional, required only if tailscale-oauth-client-id is set ``` -The `@v1.2.3` pin on the `uses:` line is the only place the Flightdeck version needs to be written: it's what gets checked out to run `ansible/deploy.yml`, and it's also the default for `flightdeck_app_ref` (the release bundle the playbook downloads and deploys) unless `extra-vars` explicitly overrides it. +The `@v1.2.3` pin on the `uses:` line is the only place the Flightdeck version needs to be written: it's what gets checked out to run `ansible/deploy.yml`, and it's also the default for `app-ref` (the release bundle the playbook downloads and deploys) unless explicitly overridden. `extra-vars` is a JSON object passed through as `ansible-playbook -e` — see [Ansible Deploy](#ansible-deploy) above for what each `flightdeck_*` key means. From 6f970f6b1549be96a1f632268c41ddea82924ceb Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:03:38 +0200 Subject: [PATCH 06/35] refactor: rename consumer workflows to target-named (hawkeye) Same reasoning as the earlier targets/keys/release-tag rename: these workflows deploy a whole target's APPS list, not one specific service. If hawkeye ever hosts a second app alongside rybbit, it's the same workflow, same target -- naming it after rubykatzen.com (the service currently there) would become misleading. Co-Authored-By: Claude Sonnet 5 --- .../workflows/{deploy-rubykatzen-com.yml => deploy-hawkeye.yml} | 2 +- .../{publish-rubykatzen-com-env.yml => publish-hawkeye-env.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{deploy-rubykatzen-com.yml => deploy-hawkeye.yml} (94%) rename .github/workflows/{publish-rubykatzen-com-env.yml => publish-hawkeye-env.yml} (89%) diff --git a/.github/workflows/deploy-rubykatzen-com.yml b/.github/workflows/deploy-hawkeye.yml similarity index 94% rename from .github/workflows/deploy-rubykatzen-com.yml rename to .github/workflows/deploy-hawkeye.yml index 9f58173..18a6e24 100644 --- a/.github/workflows/deploy-rubykatzen-com.yml +++ b/.github/workflows/deploy-hawkeye.yml @@ -1,4 +1,4 @@ -name: Deploy rubykatzen.com +name: Deploy hawkeye on: workflow_dispatch: jobs: diff --git a/.github/workflows/publish-rubykatzen-com-env.yml b/.github/workflows/publish-hawkeye-env.yml similarity index 89% rename from .github/workflows/publish-rubykatzen-com-env.yml rename to .github/workflows/publish-hawkeye-env.yml index b4d7e2d..7265ebb 100644 --- a/.github/workflows/publish-rubykatzen-com-env.yml +++ b/.github/workflows/publish-hawkeye-env.yml @@ -1,4 +1,4 @@ -name: Publish rubykatzen.com env +name: Publish hawkeye env on: workflow_dispatch: push: From 2e3f0010e47930ca8e45dd379b293595a099e897 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:16:54 +0200 Subject: [PATCH 07/35] feat: rename deploy-hawkeye.yml to deploy.yml, add a target selector One deploy entry point for the whole repo instead of one workflow file per target. Each target is its own job (currently just hawkeye); a workflow_dispatch `target` input picks which job(s) run -- "all" (default) runs every target, a specific name runs just that job. This is also the pattern we want client repos to copy: a system that knows exactly which target it just pushed config for can `gh workflow run deploy.yml -f target=` instead of redeploying the whole fleet. No scheduling added yet -- that depends on the idempotent reconciliation work in #100/#101 landing first, otherwise a cron trigger would just force-restart everything on every tick. Co-Authored-By: Claude Sonnet 5 --- .../workflows/{deploy-hawkeye.yml => deploy.yml} | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) rename .github/workflows/{deploy-hawkeye.yml => deploy.yml} (61%) diff --git a/.github/workflows/deploy-hawkeye.yml b/.github/workflows/deploy.yml similarity index 61% rename from .github/workflows/deploy-hawkeye.yml rename to .github/workflows/deploy.yml index 18a6e24..bc18357 100644 --- a/.github/workflows/deploy-hawkeye.yml +++ b/.github/workflows/deploy.yml @@ -1,8 +1,17 @@ -name: Deploy hawkeye +name: Deploy on: workflow_dispatch: + inputs: + target: + description: Deploy target to redeploy. Use "all" to redeploy every target. + type: choice + options: + - all + - hawkeye + default: all jobs: - deploy: + hawkeye: + if: inputs.target == 'all' || inputs.target == 'hawkeye' uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@main with: inventory: "100.75.50.2" From 241081db48f66e41517aa70357b6ecce04b7bd5a Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:25:33 +0200 Subject: [PATCH 08/35] feat: rename release-please.yml to release.yml, auto-trigger deploy after it - Rename release-please.yml -> release.yml (name: Release Please -> Release), matching the file. - deploy.yml now also fires on workflow_run of Release (in addition to manual workflow_dispatch). Auto-redeploys hawkeye whenever Release completes successfully -- including no-op runs where release-please found nothing to release, since workflow_run doesn't expose that workflow's own job outputs. Accepted as an MVP tradeoff: harmless once deploy.sh is idempotent (#100), just an unnecessary restart until then. - Fixed a real bug this surfaced: deploy-shared.yml's app-ref input was reused as both the checkout ref (needs a real git ref: branch, tag, SHA) and the flightdeck_app_ref bundle to deploy (can be 'latest', a GitHub Releases API concept, not a git ref). Passing app-ref: latest previously would have broken the checkout step. Split "Resolve checkout ref" (always derived from workflow_ref, used only for actions/checkout) from app-ref resolution inside "Build extra-vars" (input override, defaulting to the checkout ref if not given). deploy.yml's hawkeye job now passes app-ref: latest so it always deploys the newest release regardless of which ref runs the playbook mechanism itself. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy-shared.yml | 25 ++++++++++--------- .github/workflows/deploy.yml | 8 +++++- .../{release-please.yml => release.yml} | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) rename .github/workflows/{release-please.yml => release.yml} (98%) diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index ea5792e..5ca48bf 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -11,7 +11,7 @@ on: type: string default: root app-ref: - description: "Flightdeck ref to deploy. Defaults to the ref this workflow was called at (the @tag on the uses: line)." + description: "Release ref of the Flightdeck bundle to deploy (e.g. a version tag, or 'latest'). Defaults to the ref this workflow was called at (the @tag on the uses: line) -- pass 'latest' explicitly to always deploy the newest release regardless of which ref runs the playbook itself." type: string default: "" env-ref: @@ -53,19 +53,14 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Resolve app ref + - name: Resolve checkout ref id: ref shell: bash - env: - APP_REF_INPUT: ${{ inputs.app-ref }} run: | - ref="$APP_REF_INPUT" - if [ -z "$ref" ]; then - ref="${{ github.workflow_ref }}" - ref="${ref#*@}" - ref="${ref#refs/tags/}" - ref="${ref#refs/heads/}" - fi + ref="${{ github.workflow_ref }}" + ref="${ref#*@}" + ref="${ref#refs/tags/}" + ref="${ref#refs/heads/}" echo "ref=$ref" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v7 with: @@ -91,7 +86,8 @@ jobs: id: vars shell: bash env: - APP_REF: rubykatzen/flightdeck@${{ steps.ref.outputs.ref }} + APP_REF_INPUT: ${{ inputs.app-ref }} + CHECKOUT_REF: ${{ steps.ref.outputs.ref }} ENV_REF: ${{ inputs.env-ref }} EXTRA_REFS: ${{ inputs.extra-refs }} DEPLOY_PATH: ${{ inputs.path }} @@ -99,6 +95,11 @@ jobs: SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} SSH_USER: ${{ inputs.user }} run: | + app_ref="$APP_REF_INPUT" + if [ -z "$app_ref" ]; then + app_ref="$CHECKOUT_REF" + fi + APP_REF="rubykatzen/flightdeck@$app_ref" sops_key_file="$SOPS_KEY_FILE" if [ -z "$sops_key_file" ]; then sops_key_file="/home/$SSH_USER/.config/sops/age/keys.txt" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bc18357..82a7215 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,13 +9,19 @@ on: - all - hawkeye default: all + workflow_run: + workflows: [Release] + types: [completed] jobs: hawkeye: - if: inputs.target == 'all' || inputs.target == 'hawkeye' + if: >- + (github.event_name == 'workflow_dispatch' && (inputs.target == 'all' || inputs.target == 'hawkeye')) || + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@main with: inventory: "100.75.50.2" user: rubykatzen-com + app-ref: latest env-ref: "${{ github.repository }}@hawkeye-config:hawkeye.sops.env" tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} secrets: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release.yml similarity index 98% rename from .github/workflows/release-please.yml rename to .github/workflows/release.yml index 8a61335..9f78a35 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release Please +name: Release on: push: branches: [main] From 3f44408aa05619bea9ec3a51f8807ff2ed6a57bb Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:27:20 +0200 Subject: [PATCH 09/35] refactor!: make app-ref required, drop the checkout-ref fallback default No external consumers yet, so no reason to carry the extra branch: "if app-ref is empty, reuse whatever ref runs the playbook mechanism" was solving a DRY concern (avoid writing the version twice) at the cost of conflating two genuinely different things -- which ref checks out ansible/deploy.yml vs which release gets deployed. Now that we need them to diverge (app-ref: latest while pinned to a stable mechanism version), the implicit default is just a trap. Require app-ref explicitly instead; deploy.yml already passes it. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy-shared.yml | 12 +++--------- README.md | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 5ca48bf..a582fae 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -11,9 +11,9 @@ on: type: string default: root app-ref: - description: "Release ref of the Flightdeck bundle to deploy (e.g. a version tag, or 'latest'). Defaults to the ref this workflow was called at (the @tag on the uses: line) -- pass 'latest' explicitly to always deploy the newest release regardless of which ref runs the playbook itself." + description: "Release ref of the Flightdeck bundle to deploy (e.g. a version tag, or 'latest'). Independent of the @tag on the uses: line, which only controls which ref runs the playbook mechanism." type: string - default: "" + required: true env-ref: description: Release ref for the encrypted env package, in owner/repo@tag:asset format. type: string @@ -86,8 +86,7 @@ jobs: id: vars shell: bash env: - APP_REF_INPUT: ${{ inputs.app-ref }} - CHECKOUT_REF: ${{ steps.ref.outputs.ref }} + APP_REF: rubykatzen/flightdeck@${{ inputs.app-ref }} ENV_REF: ${{ inputs.env-ref }} EXTRA_REFS: ${{ inputs.extra-refs }} DEPLOY_PATH: ${{ inputs.path }} @@ -95,11 +94,6 @@ jobs: SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} SSH_USER: ${{ inputs.user }} run: | - app_ref="$APP_REF_INPUT" - if [ -z "$app_ref" ]; then - app_ref="$CHECKOUT_REF" - fi - APP_REF="rubykatzen/flightdeck@$app_ref" sops_key_file="$SOPS_KEY_FILE" if [ -z "$sops_key_file" ]; then sops_key_file="/home/$SSH_USER/.config/sops/age/keys.txt" diff --git a/README.md b/README.md index e2a2b07..9334f56 100644 --- a/README.md +++ b/README.md @@ -532,8 +532,8 @@ jobs: with: inventory: 100.64.0.1,100.64.0.2 # required user: root # default: root + app-ref: latest # required (e.g. a version tag, or "latest") env-ref: "${{ github.repository }}@latest:.sops.env" # required - # app-ref: v1.2.3 # optional, default: the @tag on the uses: line above # extra-refs: owner/repo@latest # optional, comma-separated, default: none # path: ~/flightdeck # optional, default shown # keep-releases: 5 # optional, default shown @@ -545,7 +545,7 @@ jobs: tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # optional, required only if tailscale-oauth-client-id is set ``` -The `@v1.2.3` pin on the `uses:` line is the only place the Flightdeck version needs to be written: it's what gets checked out to run `ansible/deploy.yml`, and it's also the default for `app-ref` (the release bundle the playbook downloads and deploys) unless explicitly overridden. +The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required — it's the release bundle the playbook downloads and deploys, and doesn't have to match the pin (e.g. pin to a stable mechanism version but pass `app-ref: latest` to always deploy the newest release). `extra-vars` is a JSON object passed through as `ansible-playbook -e` — see [Ansible Deploy](#ansible-deploy) above for what each `flightdeck_*` key means. From 8f3286e9559868b43313bce07fdb30b9b2ea30ba Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:34:07 +0200 Subject: [PATCH 10/35] feat: extract upload-bundle-shared.yml, use it from release.yml release.yml's upload job was hand-assembled from checkout + build-bundle + gh release upload steps directly, the same pattern we already moved away from for publish-sops-env. Extract it into a generic reusable workflow: checkout a ref, build a zip via the build-bundle composite action, upload it to an existing release. release.yml's own upload job now just calls it via a local path (./.github/workflows/upload-bundle-shared.yml) -- same-repo self-reference, no ref-pinning concern since it always runs from whatever commit release.yml itself runs from. Also generic enough for external use: a consumer repo packaging its own apps/ directory as an extra bundle for flightdeck_extra_refs can call it the same way (checkout defaults to the caller's own repo, same as any reusable workflow without an explicit repository: override). Docs: added README sections for both upload-bundle-shared.yml and publish-sops-env-shared.yml (the latter existed but was never documented), updated the GitHub Actions intro sentence to list all three reusable workflows, and fixed a stale release-please.yml filename left over in the project tree from the earlier rename. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 45 +++++++++------------- .github/workflows/upload-bundle-shared.yml | 37 ++++++++++++++++++ README.md | 40 +++++++++++++++++-- 3 files changed, 92 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/upload-bundle-shared.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f78a35..4b83fd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,30 +21,21 @@ jobs: upload: needs: release if: needs.release.outputs.release_created == 'true' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.release.outputs.tag_name }} - - id: bundle - uses: ./.github/actions/build-bundle - with: - bundle-name: flightdeck.zip - paths: | - apps - ansible.cfg - .env.example - backup.sh - deploy.sh - down.sh - generate-env.sh - lib.sh - logs.sh - restart.sh - up.sh - README.md - - run: gh release upload "${{ needs.release.outputs.tag_name }}" "${{ steps.bundle.outputs.bundle-path }}" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ./.github/workflows/upload-bundle-shared.yml + with: + ref: ${{ needs.release.outputs.tag_name }} + release-tag: ${{ needs.release.outputs.tag_name }} + bundle-name: flightdeck.zip + paths: | + apps + ansible.cfg + .env.example + backup.sh + deploy.sh + down.sh + generate-env.sh + lib.sh + logs.sh + restart.sh + up.sh + README.md diff --git a/.github/workflows/upload-bundle-shared.yml b/.github/workflows/upload-bundle-shared.yml new file mode 100644 index 0000000..284c877 --- /dev/null +++ b/.github/workflows/upload-bundle-shared.yml @@ -0,0 +1,37 @@ +name: Upload bundle (shared) +on: + workflow_call: + inputs: + ref: + description: Git ref to check out before building the bundle (e.g. a release tag). + type: string + required: true + release-tag: + description: Release tag to upload the bundle asset to. + type: string + required: true + bundle-name: + description: Bundle archive filename. + type: string + default: bundle.zip + paths: + description: Newline-separated paths to include in the bundle. + type: string + required: true +jobs: + upload: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + - id: bundle + uses: ./.github/actions/build-bundle + with: + bundle-name: ${{ inputs.bundle-name }} + paths: ${{ inputs.paths }} + - run: gh release upload "${{ inputs.release-tag }}" "${{ steps.bundle.outputs.bundle-path }}" --clobber + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 9334f56..29f0b5a 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ flightdeck/ │ │ ├── discover-manifest-matrix/ # Build a strategy matrix from files matching a glob │ │ └── publish-sops-env/ # Encrypt env manifest and upload to GitHub Release │ └── workflows/ -│ └── release-please.yml # Release Please + publish Flightdeck release bundle +│ └── release.yml # Release Please + publish Flightdeck release bundle │ ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template @@ -437,7 +437,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides two reusable composite actions under `.github/actions/` and one reusable workflow, `deploy-shared.yml`. +This repository provides two reusable composite actions under `.github/actions/` and three reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, and `upload-bundle-shared.yml`. --- @@ -547,7 +547,41 @@ jobs: The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required — it's the release bundle the playbook downloads and deploys, and doesn't have to match the pin (e.g. pin to a stable mechanism version but pass `app-ref: latest` to always deploy the newest release). -`extra-vars` is a JSON object passed through as `ansible-playbook -e` — see [Ansible Deploy](#ansible-deploy) above for what each `flightdeck_*` key means. +--- + +### `publish-sops-env-shared.yml` + +Wraps the `publish-sops-env` composite action with the release-existence check it requires: resolves a release tag (explicit `release-tag` input, or read from the manifest's own `release_tag` if omitted), creates that release if it doesn't exist yet, then renders/encrypts/uploads. + +```yaml +jobs: + publish: + uses: rubykatzen/flightdeck/.github/workflows/publish-sops-env-shared.yml@v1.2.3 + with: + manifest: targets/hawkeye.yml # required + # release-tag: hawkeye-config # optional, default: the manifest's own release_tag + secrets: inherit +``` + +`secrets: inherit` is required here, unlike `deploy-shared.yml`'s explicitly-named secrets: the manifest's `env:` section can reference any secret/variable name the caller has, so there's no fixed schema to declare individually — the workflow needs the caller's full secrets/vars context to resolve whatever the manifest asks for. + +--- + +### `upload-bundle-shared.yml` + +Checks out a ref, builds a zip via the `build-bundle` composite action, and uploads it to an existing GitHub Release. Generic enough for this repository's own `flightdeck.zip` release bundle, or for a consumer repository packaging just its `apps/` directory as an [extra bundle](#ansible-deploy) for `flightdeck_extra_refs`. + +```yaml +jobs: + upload: + uses: rubykatzen/flightdeck/.github/workflows/upload-bundle-shared.yml@v1.2.3 + with: + ref: v1.2.3 # required, e.g. the release tag to check out + release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) + bundle-name: apps.zip # default: bundle.zip + paths: | # required, newline-separated + apps +``` ## 📝 License From 1e192a52b71aef5f156918afad2ea7daee3eb662 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:44:17 +0200 Subject: [PATCH 11/35] refactor!: bake bundle config into upload-bundle-shared.yml, split off upload-apps-shared.yml upload-bundle-shared.yml took bundle-name/paths as caller inputs, but in practice there's exactly one caller (this repo's own release.yml) with fixed, well-known content -- flightdeck.zip is core scripts + apps/ + README, not something that varies per call. Generic parameterization here just pushed config a consumer shouldn't need to know onto the call site. Baked it in; the workflow now only takes ref/release-tag. Split the actual generic use case -- a consumer repo bundling its own apps/ as an extra bundle for flightdeck_extra_refs -- into its own purpose-built upload-apps-shared.yml (fixed to apps.zip / apps/ only). Added a comment at release.yml's upload job pointing consumers at it, so reading our own release flow makes the intended split obvious rather than inviting a copy-paste of flightdeck-specific config. Also: switched every local composite-action reference inside these shared workflows (publish-sops-env-shared.yml, upload-bundle-shared.yml, upload-apps-shared.yml) from ./.github/actions/x to $/.github/actions/x -- GitHub's new (July 2026) self-repository syntax. ./ inside a reusable workflow resolves against the CALLER's repo, which is wrong here and would have broken the moment an external repo actually called publish-sops-env-shared.yml or upload-apps-shared.yml (our own usage so far has been same-repo, so this was latent). $/ always resolves against the workflow's own defining repo at the exact ref running, regardless of caller -- matches the pattern rubykatzen/baseline already uses in lint-shared.yml. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/publish-sops-env-shared.yml | 2 +- .github/workflows/release.yml | 18 +++-------- .github/workflows/upload-apps-shared.yml | 30 +++++++++++++++++++ .github/workflows/upload-bundle-shared.yml | 26 +++++++++------- README.md | 26 +++++++++++----- 5 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/upload-apps-shared.yml diff --git a/.github/workflows/publish-sops-env-shared.yml b/.github/workflows/publish-sops-env-shared.yml index 29dfef1..832be15 100644 --- a/.github/workflows/publish-sops-env-shared.yml +++ b/.github/workflows/publish-sops-env-shared.yml @@ -44,7 +44,7 @@ jobs: --title "$TAG" \ --notes "Encrypted env release." \ --target main - - uses: ./.github/actions/publish-sops-env + - uses: $/.github/actions/publish-sops-env with: manifest: ${{ inputs.manifest }} release-tag: ${{ steps.tag.outputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b83fd8..7065bbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,10 @@ jobs: with: token: ${{ secrets.RELEASE_TOKEN }} + # This packages flightdeck's own release bundle (core scripts + apps/ + + # README). A consumer repo publishing its own apps/ as an extra bundle for + # flightdeck_extra_refs should call upload-apps-shared.yml instead, not + # copy this job -- it's pre-configured for that exact use case. upload: needs: release if: needs.release.outputs.release_created == 'true' @@ -25,17 +29,3 @@ jobs: with: ref: ${{ needs.release.outputs.tag_name }} release-tag: ${{ needs.release.outputs.tag_name }} - bundle-name: flightdeck.zip - paths: | - apps - ansible.cfg - .env.example - backup.sh - deploy.sh - down.sh - generate-env.sh - lib.sh - logs.sh - restart.sh - up.sh - README.md diff --git a/.github/workflows/upload-apps-shared.yml b/.github/workflows/upload-apps-shared.yml new file mode 100644 index 0000000..ad1e5ac --- /dev/null +++ b/.github/workflows/upload-apps-shared.yml @@ -0,0 +1,30 @@ +name: Upload apps (shared) +on: + workflow_call: + inputs: + ref: + description: Git ref to check out before building the bundle (e.g. a release tag). + type: string + required: true + release-tag: + description: Release tag to upload the bundle asset to. + type: string + required: true +jobs: + upload: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + - id: bundle + uses: $/.github/actions/build-bundle + with: + bundle-name: apps.zip + paths: | + apps + - run: gh release upload "${{ inputs.release-tag }}" "${{ steps.bundle.outputs.bundle-path }}" --clobber + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/upload-bundle-shared.yml b/.github/workflows/upload-bundle-shared.yml index 284c877..5abb8bb 100644 --- a/.github/workflows/upload-bundle-shared.yml +++ b/.github/workflows/upload-bundle-shared.yml @@ -10,14 +10,6 @@ on: description: Release tag to upload the bundle asset to. type: string required: true - bundle-name: - description: Bundle archive filename. - type: string - default: bundle.zip - paths: - description: Newline-separated paths to include in the bundle. - type: string - required: true jobs: upload: runs-on: ubuntu-latest @@ -28,10 +20,22 @@ jobs: with: ref: ${{ inputs.ref }} - id: bundle - uses: ./.github/actions/build-bundle + uses: $/.github/actions/build-bundle with: - bundle-name: ${{ inputs.bundle-name }} - paths: ${{ inputs.paths }} + bundle-name: flightdeck.zip + paths: | + apps + ansible.cfg + .env.example + backup.sh + deploy.sh + down.sh + generate-env.sh + lib.sh + logs.sh + restart.sh + up.sh + README.md - run: gh release upload "${{ inputs.release-tag }}" "${{ steps.bundle.outputs.bundle-path }}" --clobber env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 29f0b5a..e009290 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides two reusable composite actions under `.github/actions/` and three reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, and `upload-bundle-shared.yml`. +This repository provides two reusable composite actions under `.github/actions/` and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. --- @@ -569,18 +569,30 @@ jobs: ### `upload-bundle-shared.yml` -Checks out a ref, builds a zip via the `build-bundle` composite action, and uploads it to an existing GitHub Release. Generic enough for this repository's own `flightdeck.zip` release bundle, or for a consumer repository packaging just its `apps/` directory as an [extra bundle](#ansible-deploy) for `flightdeck_extra_refs`. +Checks out a ref, builds this repository's own `flightdeck.zip` release bundle (core scripts, `apps/`, README) via the `build-bundle` composite action, and uploads it to an existing GitHub Release. Used by this repository's own [`release.yml`](.github/workflows/release.yml) — the bundle contents are fixed, not configurable, since they describe what "the Flightdeck bundle" is. ```yaml jobs: upload: uses: rubykatzen/flightdeck/.github/workflows/upload-bundle-shared.yml@v1.2.3 with: - ref: v1.2.3 # required, e.g. the release tag to check out - release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) - bundle-name: apps.zip # default: bundle.zip - paths: | # required, newline-separated - apps + ref: v1.2.3 # required, e.g. the release tag to check out + release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) +``` + +--- + +### `upload-apps-shared.yml` + +Same shape as `upload-bundle-shared.yml`, but for a consumer repository packaging just its own `apps/` directory as an [extra bundle](#ansible-deploy) for `flightdeck_extra_refs` — bundle contents are fixed to `apps/` only, named `apps.zip`. + +```yaml +jobs: + upload: + uses: rubykatzen/flightdeck/.github/workflows/upload-apps-shared.yml@v1.2.3 + with: + ref: v1.2.3 # required, e.g. the release tag to check out + release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) ``` ## 📝 License From 2b7b16a41947de74881542b04d97bf34b4f8663b Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:48:30 +0200 Subject: [PATCH 12/35] refactor: publish hawkeye's encrypted env as part of the release, not a separate trigger publish-hawkeye-env.yml had its own independent trigger (push when targets/hawkeye.yml or keys/hawkeye.pub changed), decoupled from whether a release actually happened. Config changes for a target are conceptually part of the release cycle here, same as the bundle -- fold it into release.yml as another job gated on release_created, same shape as upload. Delete the standalone workflow; a per-target manual/path-based publish trigger isn't needed once every target's env publish just rides along with the release. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/publish-hawkeye-env.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/publish-hawkeye-env.yml diff --git a/.github/workflows/publish-hawkeye-env.yml b/.github/workflows/publish-hawkeye-env.yml deleted file mode 100644 index 7265ebb..0000000 --- a/.github/workflows/publish-hawkeye-env.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Publish hawkeye env -on: - workflow_dispatch: - push: - branches: [main] - paths: - - targets/hawkeye.yml - - keys/hawkeye.pub -jobs: - publish: - uses: rubykatzen/flightdeck/.github/workflows/publish-sops-env-shared.yml@main - with: - manifest: targets/hawkeye.yml - secrets: inherit From 1090efdce969d99bc560d30c8eb1da83b32fad44 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:48:49 +0200 Subject: [PATCH 13/35] feat: add hawkeye-env job to release.yml Missed staging this alongside the publish-hawkeye-env.yml removal in 2b7b16a -- this is the actual job that replaces it. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7065bbf..c701047 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,3 +29,11 @@ jobs: with: ref: ${{ needs.release.outputs.tag_name }} release-tag: ${{ needs.release.outputs.tag_name }} + + hawkeye-env: + needs: release + if: needs.release.outputs.release_created == 'true' + uses: ./.github/workflows/publish-sops-env-shared.yml + with: + manifest: targets/hawkeye.yml + secrets: inherit From d1472bb79822cf7689ac03cff8ea64f07a5eabcd Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:53:32 +0200 Subject: [PATCH 14/35] chore: remove unused discover-manifest-matrix action Predates the explicit per-target job pattern (targets/hawkeye.yml, named jobs in release.yml/deploy.yml) this repo now actually uses -- it was built for a dynamic glob-based matrix discovery approach we never ended up using here, and nothing in this repo calls it. Co-Authored-By: Claude Sonnet 5 --- .../discover-manifest-matrix/README.md | 39 ------------------- .../discover-manifest-matrix/action.yml | 26 ------------- 2 files changed, 65 deletions(-) delete mode 100644 .github/actions/discover-manifest-matrix/README.md delete mode 100644 .github/actions/discover-manifest-matrix/action.yml diff --git a/.github/actions/discover-manifest-matrix/README.md b/.github/actions/discover-manifest-matrix/README.md deleted file mode 100644 index f4dbd77..0000000 --- a/.github/actions/discover-manifest-matrix/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# discover-manifest-matrix - -Composite GitHub Action that builds a GitHub Actions strategy matrix from files matching a glob pattern. - -## Usage - -```yaml -- id: discover - uses: rubykatzen/flightdeck/.github/actions/discover-manifest-matrix@main - with: - pattern: projects/*/*.yml # required -``` - -**Output:** `matrix` — JSON object `{"manifest": ["projects/a/server.yml", ...]}`. - -Fails if no files match the pattern. - -## Example - -```yaml -jobs: - discover: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.discover.outputs.matrix }} - steps: - - uses: actions/checkout@v6 - - id: discover - uses: rubykatzen/flightdeck/.github/actions/discover-manifest-matrix@main - with: - pattern: projects/*/*.yml - - publish: - needs: discover - strategy: - matrix: ${{ fromJson(needs.discover.outputs.matrix) }} - steps: - - run: echo "${{ matrix.manifest }}" -``` diff --git a/.github/actions/discover-manifest-matrix/action.yml b/.github/actions/discover-manifest-matrix/action.yml deleted file mode 100644 index 4319f3f..0000000 --- a/.github/actions/discover-manifest-matrix/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Discover manifest matrix -description: Build a GitHub Actions strategy matrix from files matching a glob pattern. -inputs: - pattern: - description: Glob pattern to match manifest files. - required: true -outputs: - matrix: - description: JSON strategy matrix with a "manifest" key containing matched file paths. - value: ${{ steps.matrix.outputs.matrix }} -runs: - using: composite - steps: - - name: Build matrix - id: matrix - shell: bash - env: - PATTERN: ${{ inputs.pattern }} - run: | - set -euo pipefail - matrix="$(find . -path "./$PATTERN" -not -path '*/.*' | sort | sed 's|^\./||' | jq -Rsc 'split("\n") | map(select(length > 0)) | {manifest: .}')" - if [ "$(jq '.manifest | length' <<< "$matrix")" -eq 0 ]; then - echo "No files found matching: $PATTERN" >&2 - exit 1 - fi - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" From fe108e730f4cf91b2a5ca19d966c78fdf636e6f8 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:53:47 +0200 Subject: [PATCH 15/35] docs: remove discover-manifest-matrix from README Missed staging this alongside the action removal in d1472bb. Co-Authored-By: Claude Sonnet 5 --- README.md | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/README.md b/README.md index e009290..5b5afe7 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,6 @@ flightdeck/ │ └── deploy.yml # Deploy published bundle and encrypted env ├── .github/ │ ├── actions/ -│ │ ├── discover-manifest-matrix/ # Build a strategy matrix from files matching a glob │ │ └── publish-sops-env/ # Encrypt env manifest and upload to GitHub Release │ └── workflows/ │ └── release.yml # Release Please + publish Flightdeck release bundle @@ -437,44 +436,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides two reusable composite actions under `.github/actions/` and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. - ---- - -### `discover-manifest-matrix` - -Builds a GitHub Actions strategy matrix from files matching a glob pattern. - -```yaml -- id: discover - uses: rubykatzen/flightdeck/.github/actions/discover-manifest-matrix@main - with: - pattern: projects/*/*.yml # required -``` - -**Outputs:** `matrix` — JSON object `{"manifest": ["path/a.yml", "path/b.yml", ...]}`. - -**Typical use** — feed the output into a matrix job: - -```yaml -jobs: - discover: - outputs: - matrix: ${{ steps.discover.outputs.matrix }} - steps: - - uses: actions/checkout@v6 - - id: discover - uses: rubykatzen/flightdeck/.github/actions/discover-manifest-matrix@main - with: - pattern: projects/*/*.yml - - publish: - needs: discover - strategy: - matrix: ${{ fromJson(needs.discover.outputs.matrix) }} - steps: - - run: echo ${{ matrix.manifest }} -``` +This repository provides one reusable composite action under `.github/actions/` and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. --- From 74e4d02e836187be3c71631832f70c8fd7e8e16b Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 22:58:52 +0200 Subject: [PATCH 16/35] refactor: merge build-bundle and upload into one composite action build-bundle was the odd one out -- publish-sops-env already does render+encrypt+upload as one action, but build-bundle only built the zip and left the gh release upload as a separate step in each caller. Fold upload into build-bundle itself (new release-tag/token inputs), so upload-bundle-shared.yml and upload-apps-shared.yml both drop to checkout + one action call. Co-Authored-By: Claude Sonnet 5 --- .github/actions/build-bundle/action.yml | 15 ++++++++++++++- .github/workflows/upload-apps-shared.yml | 8 +++----- .github/workflows/upload-bundle-shared.yml | 8 +++----- README.md | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/actions/build-bundle/action.yml b/.github/actions/build-bundle/action.yml index 11a5273..c32e44d 100644 --- a/.github/actions/build-bundle/action.yml +++ b/.github/actions/build-bundle/action.yml @@ -1,5 +1,5 @@ name: Build app bundle -description: Build and validate a zip bundle from specified paths. +description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. inputs: paths: description: Newline-separated paths to include in the bundle. @@ -8,6 +8,12 @@ inputs: description: Bundle archive filename. required: false default: bundle.zip + release-tag: + description: Release tag to upload the bundle asset to. + required: true + token: + description: Token with permission to upload release assets. + required: true outputs: bundle-path: description: Path to the built bundle archive. @@ -40,3 +46,10 @@ runs: exit 1 fi echo "bundle-path=.bundle/$BUNDLE_NAME" >> "$GITHUB_OUTPUT" + - name: Upload bundle + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + RELEASE_TAG: ${{ inputs.release-tag }} + BUNDLE_PATH: ${{ steps.build.outputs.bundle-path }} + run: gh release upload "$RELEASE_TAG" "$BUNDLE_PATH" --clobber diff --git a/.github/workflows/upload-apps-shared.yml b/.github/workflows/upload-apps-shared.yml index ad1e5ac..89024ed 100644 --- a/.github/workflows/upload-apps-shared.yml +++ b/.github/workflows/upload-apps-shared.yml @@ -19,12 +19,10 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ inputs.ref }} - - id: bundle - uses: $/.github/actions/build-bundle + - uses: $/.github/actions/build-bundle with: bundle-name: apps.zip + release-tag: ${{ inputs.release-tag }} + token: ${{ secrets.GITHUB_TOKEN }} paths: | apps - - run: gh release upload "${{ inputs.release-tag }}" "${{ steps.bundle.outputs.bundle-path }}" --clobber - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/upload-bundle-shared.yml b/.github/workflows/upload-bundle-shared.yml index 5abb8bb..250c408 100644 --- a/.github/workflows/upload-bundle-shared.yml +++ b/.github/workflows/upload-bundle-shared.yml @@ -19,10 +19,11 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ inputs.ref }} - - id: bundle - uses: $/.github/actions/build-bundle + - uses: $/.github/actions/build-bundle with: bundle-name: flightdeck.zip + release-tag: ${{ inputs.release-tag }} + token: ${{ secrets.GITHUB_TOKEN }} paths: | apps ansible.cfg @@ -36,6 +37,3 @@ jobs: restart.sh up.sh README.md - - run: gh release upload "${{ inputs.release-tag }}" "${{ steps.bundle.outputs.bundle-path }}" --clobber - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 5b5afe7..69968c5 100644 --- a/README.md +++ b/README.md @@ -436,7 +436,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides one reusable composite action under `.github/actions/` and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. +This repository provides two composite actions under `.github/actions/` (`publish-sops-env`, documented below, and `build-bundle`, an internal implementation detail of `upload-bundle-shared.yml`/`upload-apps-shared.yml`) and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. --- From e7aed5733e7b8dba49b391b41e8860bb49606ad9 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 23:37:29 +0200 Subject: [PATCH 17/35] fix: make shared deployment workflows runnable --- .github/actionlint.yaml | 5 ++++ .github/workflows/deploy-shared.yml | 25 ++++++++----------- .github/workflows/deploy.yml | 2 +- .github/workflows/publish-sops-env-shared.yml | 1 + ansible/deploy.yml | 3 +-- 5 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..d4e3d00 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +paths: + .github/workflows/deploy-shared.yml: + # actionlint 1.7.12 predates these GitHub job context properties. + ignore: + - 'property "workflow_(repository|sha)" is not defined in object type' diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index a582fae..7c470a2 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -53,19 +53,10 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Resolve checkout ref - id: ref - shell: bash - run: | - ref="${{ github.workflow_ref }}" - ref="${ref#*@}" - ref="${ref#refs/tags/}" - ref="${ref#refs/heads/}" - echo "ref=$ref" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v7 with: - repository: rubykatzen/flightdeck - ref: ${{ steps.ref.outputs.ref }} + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - name: Install ansible-core shell: bash run: pip install --user --break-system-packages ansible-core @@ -94,19 +85,25 @@ jobs: SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} SSH_USER: ${{ inputs.user }} run: | + if [ "$SSH_USER" = root ]; then + home_dir=/root + else + home_dir="/home/$SSH_USER" + fi + deploy_path="${DEPLOY_PATH/#\~/$home_dir}" sops_key_file="$SOPS_KEY_FILE" if [ -z "$sops_key_file" ]; then - sops_key_file="/home/$SSH_USER/.config/sops/age/keys.txt" + sops_key_file="$home_dir/.config/sops/age/keys.txt" fi extra_refs_json="[]" if [ -n "$EXTRA_REFS" ]; then - extra_refs_json="$(printf '%s' "$EXTRA_REFS" | jq -R 'split(",")')" + extra_refs_json="$(printf '%s' "$EXTRA_REFS" | jq -R 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')" fi json="$(jq -n \ --arg app_ref "$APP_REF" \ --arg env_ref "$ENV_REF" \ --argjson extra_refs "$extra_refs_json" \ - --arg path "$DEPLOY_PATH" \ + --arg path "$deploy_path" \ --argjson keep_releases "$KEEP_RELEASES" \ --arg sops_key_file "$sops_key_file" \ '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_extra_refs: $extra_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 82a7215..61daf62 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,7 +19,7 @@ jobs: (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@main with: - inventory: "100.75.50.2" + inventory: "100.75.50.2," user: rubykatzen-com app-ref: latest env-ref: "${{ github.repository }}@hawkeye-config:hawkeye.sops.env" diff --git a/.github/workflows/publish-sops-env-shared.yml b/.github/workflows/publish-sops-env-shared.yml index 832be15..1b901a6 100644 --- a/.github/workflows/publish-sops-env-shared.yml +++ b/.github/workflows/publish-sops-env-shared.yml @@ -43,6 +43,7 @@ jobs: gh release create "$TAG" --repo "${{ github.repository }}" \ --title "$TAG" \ --notes "Encrypted env release." \ + --latest=false \ --target main - uses: $/.github/actions/publish-sops-env with: diff --git a/ansible/deploy.yml b/ansible/deploy.yml index cbb63a2..83a7fe0 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -12,11 +12,10 @@ - flightdeck_app_ref is defined and flightdeck_app_ref | length > 0 - flightdeck_env_ref is defined and flightdeck_env_ref | length > 0 - flightdeck_extra_refs is defined - - flightdeck_github_token | length > 0 - flightdeck_path is defined and flightdeck_path | length > 0 - flightdeck_keep_releases is defined - flightdeck_sops_age_key_file is defined and flightdeck_sops_age_key_file | length > 0 - fail_msg: "Set flightdeck_app_ref, flightdeck_env_ref, flightdeck_extra_refs, flightdeck_path, flightdeck_keep_releases, flightdeck_sops_age_key_file, and the FLIGHTDECK_GITHUB_TOKEN environment variable" + fail_msg: "Set flightdeck_app_ref, flightdeck_env_ref, flightdeck_extra_refs, flightdeck_path, flightdeck_keep_releases, and flightdeck_sops_age_key_file" - name: Apply Flightdeck paths set_fact: flightdeck_release_name: "{{ ansible_facts['date_time'].iso8601_basic_short }}" From 7558bd4b767513c309690e001c72dcdd76f40e95 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 23:41:56 +0200 Subject: [PATCH 18/35] refactor!: replace asset workflows with actions --- .github/actions/build-bundle/action.yml | 2 +- .github/actions/publish-sops-env/README.md | 6 +- .github/actions/publish-sops-env/action.yml | 73 +++++++++-------- .github/workflows/publish-sops-env-shared.yml | 55 ------------- .github/workflows/release.yml | 46 ++++++++--- .github/workflows/upload-apps-shared.yml | 28 ------- .github/workflows/upload-bundle-shared.yml | 39 --------- README.md | 81 +++++++------------ 8 files changed, 103 insertions(+), 227 deletions(-) delete mode 100644 .github/workflows/publish-sops-env-shared.yml delete mode 100644 .github/workflows/upload-apps-shared.yml delete mode 100644 .github/workflows/upload-bundle-shared.yml diff --git a/.github/actions/build-bundle/action.yml b/.github/actions/build-bundle/action.yml index c32e44d..c32e890 100644 --- a/.github/actions/build-bundle/action.yml +++ b/.github/actions/build-bundle/action.yml @@ -1,4 +1,4 @@ -name: Build app bundle +name: Build and upload bundle description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. inputs: paths: diff --git a/.github/actions/publish-sops-env/README.md b/.github/actions/publish-sops-env/README.md index 4f0765c..d942e8a 100644 --- a/.github/actions/publish-sops-env/README.md +++ b/.github/actions/publish-sops-env/README.md @@ -1,8 +1,6 @@ # publish-sops-env -Composite GitHub Action that renders an env manifest from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` as a GitHub Release asset. - -The release must exist before this action runs. Create it in a dedicated job and pass the tag explicitly via `release-tag`. +Composite GitHub Action that renders an env manifest from GitHub Secrets/Variables, encrypts it for named age recipients, ensures the target release exists, and uploads `.sops.env` as a GitHub Release asset. ## Usage @@ -11,7 +9,7 @@ The release must exist before this action runs. Create it in a dedicated job and with: manifest: projects/flightdeck/mainframe.yml # required keys-directory: keys # default: keys - release-tag: ${{ needs.release.outputs.tag }} # default: manifest release_tag or repo name + release-tag: ${{ needs.release.outputs.tag }} # default: manifest release_tag or latest release-repo: "" # default: current repository asset-name: "" # default: manifest release_asset or .sops.env token: ${{ secrets.GITHUB_TOKEN }} # required diff --git a/.github/actions/publish-sops-env/action.yml b/.github/actions/publish-sops-env/action.yml index 9f7df9e..9a7a6e7 100644 --- a/.github/actions/publish-sops-env/action.yml +++ b/.github/actions/publish-sops-env/action.yml @@ -9,7 +9,7 @@ inputs: required: false default: "" release-tag: - description: Release tag to publish. Defaults to manifest release_tag or current repository name. + description: Release tag to publish. Defaults to manifest release_tag or latest. required: false default: "" asset-name: @@ -31,7 +31,7 @@ outputs: description: Release tag used for publication. value: ${{ steps.publish.outputs.tag }} ref: - description: Short release ref in owner/repo@tag format. + description: Release asset ref in owner/repo@tag:asset format. value: ${{ steps.publish.outputs.ref }} runs: using: composite @@ -58,6 +58,36 @@ runs: python3 "$GITHUB_ACTION_PATH/scripts/render-env.py" \ --manifest "$MANIFEST" \ --output "$RUNNER_TEMP/publish-sops-env/plain.env" + - name: Resolve release target + id: target + shell: bash + env: + INPUT_RELEASE_REPO: ${{ inputs.release-repo }} + MANIFEST_RELEASE_REPO: ${{ steps.render.outputs.release_repo }} + DEFAULT_RELEASE_REPO: ${{ github.repository }} + INPUT_RELEASE_TAG: ${{ inputs.release-tag }} + MANIFEST_RELEASE_TAG: ${{ steps.render.outputs.release_tag }} + INPUT_ASSET_NAME: ${{ inputs.asset-name }} + MANIFEST_ASSET_NAME: ${{ steps.render.outputs.release_asset }} + run: | + release_repo="${INPUT_RELEASE_REPO:-${MANIFEST_RELEASE_REPO:-$DEFAULT_RELEASE_REPO}}" + tag="${INPUT_RELEASE_TAG:-${MANIFEST_RELEASE_TAG:-latest}}" + asset_name="${INPUT_ASSET_NAME:-$MANIFEST_ASSET_NAME}" + echo "release_repo=$release_repo" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "asset_name=$asset_name" >> "$GITHUB_OUTPUT" + - name: Ensure release exists + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + RELEASE_REPO: ${{ steps.target.outputs.release_repo }} + TAG: ${{ steps.target.outputs.tag }} + run: | + gh release view "$TAG" --repo "$RELEASE_REPO" >/dev/null 2>&1 || \ + gh release create "$TAG" --repo "$RELEASE_REPO" \ + --title "$TAG" \ + --notes "Encrypted env release." \ + --latest=false - name: Encrypt env shell: bash env: @@ -84,41 +114,18 @@ runs: id: publish shell: bash env: - INPUT_RELEASE_REPO: ${{ inputs.release-repo }} - MANIFEST_RELEASE_REPO: ${{ steps.render.outputs.release_repo }} - DEFAULT_RELEASE_REPO: ${{ github.repository }} - DEFAULT_RELEASE_TAG: latest - INPUT_RELEASE_TAG: ${{ inputs.release-tag }} - MANIFEST_RELEASE_TAG: ${{ steps.render.outputs.release_tag }} - INPUT_ASSET_NAME: ${{ inputs.asset-name }} - MANIFEST_ASSET_NAME: ${{ steps.render.outputs.release_asset }} + RELEASE_REPO: ${{ steps.target.outputs.release_repo }} + TAG: ${{ steps.target.outputs.tag }} + ASSET_NAME: ${{ steps.target.outputs.asset_name }} GH_TOKEN: ${{ inputs.token }} run: | set -euo pipefail - release_repo="$INPUT_RELEASE_REPO" - if [ -z "$release_repo" ]; then - release_repo="$MANIFEST_RELEASE_REPO" - fi - if [ -z "$release_repo" ]; then - release_repo="$DEFAULT_RELEASE_REPO" - fi - tag="$INPUT_RELEASE_TAG" - if [ -z "$tag" ]; then - tag="$MANIFEST_RELEASE_TAG" - fi - if [ -z "$tag" ]; then - tag="$DEFAULT_RELEASE_TAG" - fi - asset_name="$INPUT_ASSET_NAME" - if [ -z "$asset_name" ]; then - asset_name="$MANIFEST_ASSET_NAME" - fi - asset_path="$RUNNER_TEMP/publish-sops-env/$asset_name" + asset_path="$RUNNER_TEMP/publish-sops-env/$ASSET_NAME" cp "$RUNNER_TEMP/publish-sops-env/.sops.env" "$asset_path" - gh release upload "$tag" "$asset_path" --repo "$release_repo" --clobber - echo "release_repo=$release_repo" >> "$GITHUB_OUTPUT" - echo "tag=$tag" >> "$GITHUB_OUTPUT" - echo "ref=$release_repo@$tag:$asset_name" >> "$GITHUB_OUTPUT" + gh release upload "$TAG" "$asset_path" --repo "$RELEASE_REPO" --clobber + echo "release_repo=$RELEASE_REPO" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "ref=$RELEASE_REPO@$TAG:$ASSET_NAME" >> "$GITHUB_OUTPUT" - name: Cleanup if: always() shell: bash diff --git a/.github/workflows/publish-sops-env-shared.yml b/.github/workflows/publish-sops-env-shared.yml deleted file mode 100644 index 1b901a6..0000000 --- a/.github/workflows/publish-sops-env-shared.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Publish SOPS env (shared) -on: - workflow_call: - inputs: - manifest: - description: Path to the YAML env manifest. - type: string - required: true - release-tag: - description: Release tag to ensure exists and publish the encrypted asset to. Defaults to the manifest's own release_tag. - type: string - default: "" -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - - name: Resolve release tag - id: tag - shell: bash - env: - INPUT_TAG: ${{ inputs.release-tag }} - MANIFEST: ${{ inputs.manifest }} - run: | - tag="$INPUT_TAG" - if [ -z "$tag" ]; then - tag="$(grep -m1 '^release_tag:' "$MANIFEST" | sed 's/^release_tag:[[:space:]]*//')" - fi - if [ -z "$tag" ]; then - echo "::error::No release-tag input and no release_tag in $MANIFEST" >&2 - exit 1 - fi - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - name: Ensure release exists - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ steps.tag.outputs.tag }} - run: | - gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create "$TAG" --repo "${{ github.repository }}" \ - --title "$TAG" \ - --notes "Encrypted env release." \ - --latest=false \ - --target main - - uses: $/.github/actions/publish-sops-env - with: - manifest: ${{ inputs.manifest }} - release-tag: ${{ steps.tag.outputs.tag }} - token: ${{ secrets.GITHUB_TOKEN }} - env: - GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} - GITHUB_VARS_JSON: ${{ toJson(vars) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c701047..75b9669 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,22 +18,42 @@ jobs: with: token: ${{ secrets.RELEASE_TOKEN }} - # This packages flightdeck's own release bundle (core scripts + apps/ + - # README). A consumer repo publishing its own apps/ as an extra bundle for - # flightdeck_extra_refs should call upload-apps-shared.yml instead, not - # copy this job -- it's pre-configured for that exact use case. upload: needs: release if: needs.release.outputs.release_created == 'true' - uses: ./.github/workflows/upload-bundle-shared.yml - with: - ref: ${{ needs.release.outputs.tag_name }} - release-tag: ${{ needs.release.outputs.tag_name }} - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.release.outputs.tag_name }} + - uses: $/.github/actions/build-bundle + with: + bundle-name: flightdeck.zip + release-tag: ${{ needs.release.outputs.tag_name }} + token: ${{ secrets.GITHUB_TOKEN }} + paths: | + apps + ansible.cfg + .env.example + backup.sh + deploy.sh + down.sh + generate-env.sh + lib.sh + logs.sh + restart.sh + up.sh + README.md hawkeye-env: needs: release if: needs.release.outputs.release_created == 'true' - uses: ./.github/workflows/publish-sops-env-shared.yml - with: - manifest: targets/hawkeye.yml - secrets: inherit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: $/.github/actions/publish-sops-env + with: + manifest: targets/hawkeye.yml + token: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} + GITHUB_VARS_JSON: ${{ toJson(vars) }} diff --git a/.github/workflows/upload-apps-shared.yml b/.github/workflows/upload-apps-shared.yml deleted file mode 100644 index 89024ed..0000000 --- a/.github/workflows/upload-apps-shared.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Upload apps (shared) -on: - workflow_call: - inputs: - ref: - description: Git ref to check out before building the bundle (e.g. a release tag). - type: string - required: true - release-tag: - description: Release tag to upload the bundle asset to. - type: string - required: true -jobs: - upload: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ inputs.ref }} - - uses: $/.github/actions/build-bundle - with: - bundle-name: apps.zip - release-tag: ${{ inputs.release-tag }} - token: ${{ secrets.GITHUB_TOKEN }} - paths: | - apps diff --git a/.github/workflows/upload-bundle-shared.yml b/.github/workflows/upload-bundle-shared.yml deleted file mode 100644 index 250c408..0000000 --- a/.github/workflows/upload-bundle-shared.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Upload bundle (shared) -on: - workflow_call: - inputs: - ref: - description: Git ref to check out before building the bundle (e.g. a release tag). - type: string - required: true - release-tag: - description: Release tag to upload the bundle asset to. - type: string - required: true -jobs: - upload: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ inputs.ref }} - - uses: $/.github/actions/build-bundle - with: - bundle-name: flightdeck.zip - release-tag: ${{ inputs.release-tag }} - token: ${{ secrets.GITHUB_TOKEN }} - paths: | - apps - ansible.cfg - .env.example - backup.sh - deploy.sh - down.sh - generate-env.sh - lib.sh - logs.sh - restart.sh - up.sh - README.md diff --git a/README.md b/README.md index 69968c5..bd1a550 100644 --- a/README.md +++ b/README.md @@ -173,9 +173,11 @@ flightdeck/ │ └── deploy.yml # Deploy published bundle and encrypted env ├── .github/ │ ├── actions/ +│ │ ├── build-bundle/ # Build and upload a release bundle │ │ └── publish-sops-env/ # Encrypt env manifest and upload to GitHub Release │ └── workflows/ -│ └── release.yml # Release Please + publish Flightdeck release bundle +│ ├── deploy-shared.yml # Reusable deployment workflow +│ └── release.yml # Release Please + publish Flightdeck assets │ ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template @@ -436,22 +438,20 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides two composite actions under `.github/actions/` (`publish-sops-env`, documented below, and `build-bundle`, an internal implementation detail of `upload-bundle-shared.yml`/`upload-apps-shared.yml`) and four reusable workflows: `deploy-shared.yml`, `publish-sops-env-shared.yml`, `upload-bundle-shared.yml`, and `upload-apps-shared.yml`. +This repository provides two composite actions under `.github/actions/` (`build-bundle` and `publish-sops-env`) and one reusable workflow, `deploy-shared.yml`. --- ### `publish-sops-env` -Renders an env manifest from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` as a GitHub Release asset. - -The release must already exist before this action runs. Create it in a separate job and pass the tag explicitly. +Renders an env manifest from GitHub Secrets/Variables, encrypts it with SOPS age recipients, ensures the target release exists, and uploads `.sops.env` as a GitHub Release asset. ```yaml - uses: rubykatzen/flightdeck/.github/actions/publish-sops-env@main with: manifest: projects/flightdeck/mainframe.yml # required keys-directory: keys # default: keys - release-tag: latest # default: manifest release_tag or repo name + release-tag: latest # default: manifest release_tag or latest release-repo: "" # default: current repository asset-name: "" # default: manifest release_asset or .sops.env token: ${{ secrets.GITHUB_TOKEN }} # required @@ -479,6 +479,27 @@ Secrets take precedence over Variables when both contain the same source key. Ev --- +### `build-bundle` + +Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. Callers choose the archive name and contents, so the same action publishes the core `flightdeck.zip` bundle or a consumer repository's `flightdeck-extra.zip` bundle. + +```yaml +steps: + - uses: actions/checkout@v7 + with: + ref: v1.2.3 + - uses: rubykatzen/flightdeck/.github/actions/build-bundle@v1.2.3 + with: + paths: apps + bundle-name: flightdeck-extra.zip + release-tag: v1.2.3 + token: ${{ secrets.GITHUB_TOKEN }} +``` + +Requires `contents: write` permission on the calling job. `flightdeck-extra.zip` is the default asset name expected by `flightdeck_extra_refs`; use an explicit `:asset-name` suffix in the ref when publishing a different filename. + +--- + ### `deploy-shared.yml` Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied inventory. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `env-ref` typically references that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. @@ -509,54 +530,6 @@ jobs: The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required — it's the release bundle the playbook downloads and deploys, and doesn't have to match the pin (e.g. pin to a stable mechanism version but pass `app-ref: latest` to always deploy the newest release). ---- - -### `publish-sops-env-shared.yml` - -Wraps the `publish-sops-env` composite action with the release-existence check it requires: resolves a release tag (explicit `release-tag` input, or read from the manifest's own `release_tag` if omitted), creates that release if it doesn't exist yet, then renders/encrypts/uploads. - -```yaml -jobs: - publish: - uses: rubykatzen/flightdeck/.github/workflows/publish-sops-env-shared.yml@v1.2.3 - with: - manifest: targets/hawkeye.yml # required - # release-tag: hawkeye-config # optional, default: the manifest's own release_tag - secrets: inherit -``` - -`secrets: inherit` is required here, unlike `deploy-shared.yml`'s explicitly-named secrets: the manifest's `env:` section can reference any secret/variable name the caller has, so there's no fixed schema to declare individually — the workflow needs the caller's full secrets/vars context to resolve whatever the manifest asks for. - ---- - -### `upload-bundle-shared.yml` - -Checks out a ref, builds this repository's own `flightdeck.zip` release bundle (core scripts, `apps/`, README) via the `build-bundle` composite action, and uploads it to an existing GitHub Release. Used by this repository's own [`release.yml`](.github/workflows/release.yml) — the bundle contents are fixed, not configurable, since they describe what "the Flightdeck bundle" is. - -```yaml -jobs: - upload: - uses: rubykatzen/flightdeck/.github/workflows/upload-bundle-shared.yml@v1.2.3 - with: - ref: v1.2.3 # required, e.g. the release tag to check out - release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) -``` - ---- - -### `upload-apps-shared.yml` - -Same shape as `upload-bundle-shared.yml`, but for a consumer repository packaging just its own `apps/` directory as an [extra bundle](#ansible-deploy) for `flightdeck_extra_refs` — bundle contents are fixed to `apps/` only, named `apps.zip`. - -```yaml -jobs: - upload: - uses: rubykatzen/flightdeck/.github/workflows/upload-apps-shared.yml@v1.2.3 - with: - ref: v1.2.3 # required, e.g. the release tag to check out - release-tag: v1.2.3 # required, the release to attach the asset to (must already exist) -``` - ## 📝 License This project is provided as-is for self-hosted deployment. From 14f7e14139147c5ecf9edb36d867c90da8e5ef32 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 18 Aug 2026 23:44:38 +0200 Subject: [PATCH 19/35] refactor: publish env to the created release --- .github/actions/publish-sops-env/README.md | 6 +++-- .github/actions/publish-sops-env/action.yml | 23 +++---------------- .../publish-sops-env/scripts/render-env.py | 9 ++------ .github/workflows/deploy.yml | 2 +- .github/workflows/release.yml | 1 + README.md | 4 ++-- targets/hawkeye.yml | 1 - 7 files changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/actions/publish-sops-env/README.md b/.github/actions/publish-sops-env/README.md index d942e8a..ea4b28a 100644 --- a/.github/actions/publish-sops-env/README.md +++ b/.github/actions/publish-sops-env/README.md @@ -1,6 +1,8 @@ # publish-sops-env -Composite GitHub Action that renders an env manifest from GitHub Secrets/Variables, encrypts it for named age recipients, ensures the target release exists, and uploads `.sops.env` as a GitHub Release asset. +Composite GitHub Action that renders an env manifest from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` to an existing GitHub Release. + +The release must exist before this action runs. ## Usage @@ -9,7 +11,7 @@ Composite GitHub Action that renders an env manifest from GitHub Secrets/Variabl with: manifest: projects/flightdeck/mainframe.yml # required keys-directory: keys # default: keys - release-tag: ${{ needs.release.outputs.tag }} # default: manifest release_tag or latest + release-tag: ${{ needs.release.outputs.tag }} # required, must already exist release-repo: "" # default: current repository asset-name: "" # default: manifest release_asset or .sops.env token: ${{ secrets.GITHUB_TOKEN }} # required diff --git a/.github/actions/publish-sops-env/action.yml b/.github/actions/publish-sops-env/action.yml index 9a7a6e7..e5b020c 100644 --- a/.github/actions/publish-sops-env/action.yml +++ b/.github/actions/publish-sops-env/action.yml @@ -9,9 +9,8 @@ inputs: required: false default: "" release-tag: - description: Release tag to publish. Defaults to manifest release_tag or latest. - required: false - default: "" + description: Existing release tag to publish the asset to. + required: true asset-name: description: Release asset name. Defaults to manifest release_asset or .sops.env. required: false @@ -65,29 +64,13 @@ runs: INPUT_RELEASE_REPO: ${{ inputs.release-repo }} MANIFEST_RELEASE_REPO: ${{ steps.render.outputs.release_repo }} DEFAULT_RELEASE_REPO: ${{ github.repository }} - INPUT_RELEASE_TAG: ${{ inputs.release-tag }} - MANIFEST_RELEASE_TAG: ${{ steps.render.outputs.release_tag }} INPUT_ASSET_NAME: ${{ inputs.asset-name }} MANIFEST_ASSET_NAME: ${{ steps.render.outputs.release_asset }} run: | release_repo="${INPUT_RELEASE_REPO:-${MANIFEST_RELEASE_REPO:-$DEFAULT_RELEASE_REPO}}" - tag="${INPUT_RELEASE_TAG:-${MANIFEST_RELEASE_TAG:-latest}}" asset_name="${INPUT_ASSET_NAME:-$MANIFEST_ASSET_NAME}" echo "release_repo=$release_repo" >> "$GITHUB_OUTPUT" - echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "asset_name=$asset_name" >> "$GITHUB_OUTPUT" - - name: Ensure release exists - shell: bash - env: - GH_TOKEN: ${{ inputs.token }} - RELEASE_REPO: ${{ steps.target.outputs.release_repo }} - TAG: ${{ steps.target.outputs.tag }} - run: | - gh release view "$TAG" --repo "$RELEASE_REPO" >/dev/null 2>&1 || \ - gh release create "$TAG" --repo "$RELEASE_REPO" \ - --title "$TAG" \ - --notes "Encrypted env release." \ - --latest=false - name: Encrypt env shell: bash env: @@ -115,7 +98,7 @@ runs: shell: bash env: RELEASE_REPO: ${{ steps.target.outputs.release_repo }} - TAG: ${{ steps.target.outputs.tag }} + TAG: ${{ inputs.release-tag }} ASSET_NAME: ${{ steps.target.outputs.asset_name }} GH_TOKEN: ${{ inputs.token }} run: | diff --git a/.github/actions/publish-sops-env/scripts/render-env.py b/.github/actions/publish-sops-env/scripts/render-env.py index 2c9f177..87abf63 100644 --- a/.github/actions/publish-sops-env/scripts/render-env.py +++ b/.github/actions/publish-sops-env/scripts/render-env.py @@ -13,7 +13,6 @@ SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") RELEASE_REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") -RELEASE_TAG_RE = re.compile(r"^[A-Za-z0-9_.-]+$") RELEASE_ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") @@ -61,8 +60,9 @@ def load_manifest(path): raise ManifestError(f"{path} must contain a YAML mapping") if "package" in manifest: raise ManifestError("package has been replaced by release-repo/release-tag GitHub Releases configuration") + if "release_tag" in manifest: + raise ManifestError("release_tag must be passed to the action, not stored in the manifest") release_repo = manifest.get("release_repo") - release_tag = manifest.get("release_tag") release_asset = manifest.get("release_asset", f"{path.stem}.sops.env") keys = manifest.get("keys") env = manifest.get("env") @@ -72,10 +72,6 @@ def load_manifest(path): not isinstance(release_repo, str) or not RELEASE_REPO_RE.fullmatch(release_repo) ): raise ManifestError("release_repo must be in owner/repo format") - if release_tag is not None and ( - not isinstance(release_tag, str) or not RELEASE_TAG_RE.fullmatch(release_tag) - ): - raise ManifestError("release_tag must contain only letters, numbers, dots, underscores, or hyphens") if not isinstance(release_asset, str) or not RELEASE_ASSET_RE.fullmatch(release_asset): raise ManifestError("release_asset must be named like server.sops.env") if not isinstance(keys, list) or not keys: @@ -149,7 +145,6 @@ def main(argv=None): write_github_outputs( { "release_repo": manifest.get("release_repo", ""), - "release_tag": manifest.get("release_tag", ""), "release_asset": manifest.get("release_asset", f"{args.manifest.stem}.sops.env"), "keys": ",".join(manifest["keys"]), } diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 61daf62..038de40 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,7 +22,7 @@ jobs: inventory: "100.75.50.2," user: rubykatzen-com app-ref: latest - env-ref: "${{ github.repository }}@hawkeye-config:hawkeye.sops.env" + env-ref: "${{ github.repository }}@latest:hawkeye.sops.env" tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} secrets: ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75b9669..379b66b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,7 @@ jobs: - uses: $/.github/actions/publish-sops-env with: manifest: targets/hawkeye.yml + release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} env: GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} diff --git a/README.md b/README.md index bd1a550..54aaa17 100644 --- a/README.md +++ b/README.md @@ -444,14 +444,14 @@ This repository provides two composite actions under `.github/actions/` (`build- ### `publish-sops-env` -Renders an env manifest from GitHub Secrets/Variables, encrypts it with SOPS age recipients, ensures the target release exists, and uploads `.sops.env` as a GitHub Release asset. +Renders an env manifest from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` to an existing GitHub Release. Release creation remains the calling workflow's responsibility. ```yaml - uses: rubykatzen/flightdeck/.github/actions/publish-sops-env@main with: manifest: projects/flightdeck/mainframe.yml # required keys-directory: keys # default: keys - release-tag: latest # default: manifest release_tag or latest + release-tag: latest # required, must already exist release-repo: "" # default: current repository asset-name: "" # default: manifest release_asset or .sops.env token: ${{ secrets.GITHUB_TOKEN }} # required diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 688d2b1..74c2ac1 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,4 +1,3 @@ -release_tag: hawkeye-config release_asset: hawkeye.sops.env keys: From a18ec62058d9b0924f0a095db83c7344a6c4c877 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 01:39:34 +0200 Subject: [PATCH 20/35] feat: drive encryption and deploys from targets --- .github/actions/encrypt-env/README.md | 42 +++ .../action.yml | 45 ++-- .../requirements.txt | 0 .../scripts/render-env.py | 35 +-- .../tests/test_render_env.py | 29 ++- .github/actions/load-targets/README.md | 18 ++ .github/actions/load-targets/action.yml | 35 +++ .github/actions/load-targets/requirements.txt | 1 + .../load-targets/scripts/load-targets.py | 241 ++++++++++++++++++ .../load-targets/tests/test_load_targets.py | 95 +++++++ .github/actions/publish-sops-env/README.md | 44 ---- .github/workflows/deploy-shared.yml | 23 +- .github/workflows/deploy.yml | 47 +++- .github/workflows/release.yml | 26 +- AGENTS.md | 2 +- README.md | 74 ++++-- targets/hawkeye.yml | 47 ++-- 17 files changed, 635 insertions(+), 169 deletions(-) create mode 100644 .github/actions/encrypt-env/README.md rename .github/actions/{publish-sops-env => encrypt-env}/action.yml (66%) rename .github/actions/{publish-sops-env => encrypt-env}/requirements.txt (100%) rename .github/actions/{publish-sops-env => encrypt-env}/scripts/render-env.py (77%) rename .github/actions/{publish-sops-env => encrypt-env}/tests/test_render_env.py (79%) create mode 100644 .github/actions/load-targets/README.md create mode 100644 .github/actions/load-targets/action.yml create mode 100644 .github/actions/load-targets/requirements.txt create mode 100644 .github/actions/load-targets/scripts/load-targets.py create mode 100644 .github/actions/load-targets/tests/test_load_targets.py delete mode 100644 .github/actions/publish-sops-env/README.md diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md new file mode 100644 index 0000000..f637368 --- /dev/null +++ b/.github/actions/encrypt-env/README.md @@ -0,0 +1,42 @@ +# encrypt-env + +Composite GitHub Action that renders the `encrypt` section of a target from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` to an existing GitHub Release. + +The release must exist before this action runs. + +## Usage + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main + with: + manifest: targets/mainframe.yml # required + keys-directory: keys # default: keys + release-tag: ${{ needs.release.outputs.tag }} # required, must already exist + release-repo: "" # default: current repository + token: ${{ secrets.GITHUB_TOKEN }} # required + env: + GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} + GITHUB_VARS_JSON: ${{ toJson(vars) }} +``` + +The calling job requires: + +```yaml +permissions: + contents: write +``` + +## Manifest + +```yaml +encrypt: + asset: mainframe.sops.env + keys: + - mainframe + env: + APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name + APPS_TIMEZONE: APPS_TIMEZONE + APPS: APPS_MAINFRAME +``` + +For each name in `encrypt.keys`, the action loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. diff --git a/.github/actions/publish-sops-env/action.yml b/.github/actions/encrypt-env/action.yml similarity index 66% rename from .github/actions/publish-sops-env/action.yml rename to .github/actions/encrypt-env/action.yml index e5b020c..d694223 100644 --- a/.github/actions/publish-sops-env/action.yml +++ b/.github/actions/encrypt-env/action.yml @@ -1,20 +1,16 @@ -name: Publish SOPS env -description: Render an env manifest, encrypt it with SOPS age recipients, and publish it as a GitHub Release asset. +name: Encrypt env +description: Render a target env, encrypt it with SOPS age recipients, and upload it to a GitHub Release. inputs: manifest: description: Path to the YAML env manifest. required: true release-repo: - description: GitHub repository to publish the release asset to. Defaults to the current repository or manifest release_repo. + description: GitHub repository containing the release. Defaults to the current repository. required: false default: "" release-tag: description: Existing release tag to publish the asset to. required: true - asset-name: - description: Release asset name. Defaults to manifest release_asset or .sops.env. - required: false - default: "" keys-directory: description: Directory containing public age recipients named .pub. required: false @@ -41,36 +37,33 @@ runs: SOPS_VERSION: "3.13.1" run: | python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" - mkdir -p "$RUNNER_TEMP/publish-sops-env/bin" + mkdir -p "$RUNNER_TEMP/encrypt-env/bin" curl --fail --location --silent --show-error \ "https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64" \ - --output "$RUNNER_TEMP/publish-sops-env/bin/sops" - chmod +x "$RUNNER_TEMP/publish-sops-env/bin/sops" - echo "$RUNNER_TEMP/publish-sops-env/bin" >> "$GITHUB_PATH" + --output "$RUNNER_TEMP/encrypt-env/bin/sops" + chmod +x "$RUNNER_TEMP/encrypt-env/bin/sops" + echo "$RUNNER_TEMP/encrypt-env/bin" >> "$GITHUB_PATH" - name: Render env id: render shell: bash env: MANIFEST: ${{ inputs.manifest }} run: | - mkdir -p "$RUNNER_TEMP/publish-sops-env" + mkdir -p "$RUNNER_TEMP/encrypt-env" python3 "$GITHUB_ACTION_PATH/scripts/render-env.py" \ --manifest "$MANIFEST" \ - --output "$RUNNER_TEMP/publish-sops-env/plain.env" + --output "$RUNNER_TEMP/encrypt-env/plain.env" - name: Resolve release target id: target shell: bash env: INPUT_RELEASE_REPO: ${{ inputs.release-repo }} - MANIFEST_RELEASE_REPO: ${{ steps.render.outputs.release_repo }} DEFAULT_RELEASE_REPO: ${{ github.repository }} - INPUT_ASSET_NAME: ${{ inputs.asset-name }} - MANIFEST_ASSET_NAME: ${{ steps.render.outputs.release_asset }} + ASSET_NAME: ${{ steps.render.outputs.asset }} run: | - release_repo="${INPUT_RELEASE_REPO:-${MANIFEST_RELEASE_REPO:-$DEFAULT_RELEASE_REPO}}" - asset_name="${INPUT_ASSET_NAME:-$MANIFEST_ASSET_NAME}" + release_repo="${INPUT_RELEASE_REPO:-$DEFAULT_RELEASE_REPO}" echo "release_repo=$release_repo" >> "$GITHUB_OUTPUT" - echo "asset_name=$asset_name" >> "$GITHUB_OUTPUT" + echo "asset_name=$ASSET_NAME" >> "$GITHUB_OUTPUT" - name: Encrypt env shell: bash env: @@ -90,10 +83,10 @@ runs: SOPS_AGE_RECIPIENTS="$recipients" sops encrypt \ --input-type dotenv \ --output-type dotenv \ - "$RUNNER_TEMP/publish-sops-env/plain.env" \ - > "$RUNNER_TEMP/publish-sops-env/.sops.env" - rm "$RUNNER_TEMP/publish-sops-env/plain.env" - - name: Publish release asset + "$RUNNER_TEMP/encrypt-env/plain.env" \ + > "$RUNNER_TEMP/encrypt-env/.sops.env" + rm "$RUNNER_TEMP/encrypt-env/plain.env" + - name: Upload encrypted env id: publish shell: bash env: @@ -103,8 +96,8 @@ runs: GH_TOKEN: ${{ inputs.token }} run: | set -euo pipefail - asset_path="$RUNNER_TEMP/publish-sops-env/$ASSET_NAME" - cp "$RUNNER_TEMP/publish-sops-env/.sops.env" "$asset_path" + asset_path="$RUNNER_TEMP/encrypt-env/$ASSET_NAME" + cp "$RUNNER_TEMP/encrypt-env/.sops.env" "$asset_path" gh release upload "$TAG" "$asset_path" --repo "$RELEASE_REPO" --clobber echo "release_repo=$RELEASE_REPO" >> "$GITHUB_OUTPUT" echo "tag=$TAG" >> "$GITHUB_OUTPUT" @@ -112,4 +105,4 @@ runs: - name: Cleanup if: always() shell: bash - run: rm -rf "$RUNNER_TEMP/publish-sops-env" + run: rm -rf "$RUNNER_TEMP/encrypt-env" diff --git a/.github/actions/publish-sops-env/requirements.txt b/.github/actions/encrypt-env/requirements.txt similarity index 100% rename from .github/actions/publish-sops-env/requirements.txt rename to .github/actions/encrypt-env/requirements.txt diff --git a/.github/actions/publish-sops-env/scripts/render-env.py b/.github/actions/encrypt-env/scripts/render-env.py similarity index 77% rename from .github/actions/publish-sops-env/scripts/render-env.py rename to .github/actions/encrypt-env/scripts/render-env.py index 87abf63..2dd7f0b 100644 --- a/.github/actions/publish-sops-env/scripts/render-env.py +++ b/.github/actions/encrypt-env/scripts/render-env.py @@ -12,8 +12,7 @@ ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") -RELEASE_REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") -RELEASE_ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") +ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") class ManifestError(Exception): @@ -58,22 +57,17 @@ def load_manifest(path): raise ManifestError(f"{path} is not valid YAML: {exc}") from exc if not isinstance(manifest, dict): raise ManifestError(f"{path} must contain a YAML mapping") - if "package" in manifest: - raise ManifestError("package has been replaced by release-repo/release-tag GitHub Releases configuration") - if "release_tag" in manifest: - raise ManifestError("release_tag must be passed to the action, not stored in the manifest") - release_repo = manifest.get("release_repo") - release_asset = manifest.get("release_asset", f"{path.stem}.sops.env") - keys = manifest.get("keys") - env = manifest.get("env") - if "raw_env" in manifest: - raise ManifestError("raw_env is no longer supported; store values as data, not shell syntax") - if release_repo is not None and ( - not isinstance(release_repo, str) or not RELEASE_REPO_RE.fullmatch(release_repo) - ): - raise ManifestError("release_repo must be in owner/repo format") - if not isinstance(release_asset, str) or not RELEASE_ASSET_RE.fullmatch(release_asset): - raise ManifestError("release_asset must be named like server.sops.env") + encrypt = manifest.get("encrypt") + if not isinstance(encrypt, dict): + raise ManifestError(f"{path} must contain an encrypt mapping") + unknown = sorted(set(encrypt) - {"asset", "keys", "env"}) + if unknown: + raise ManifestError("encrypt contains unknown keys: " + ", ".join(unknown)) + asset = encrypt.get("asset") + keys = encrypt.get("keys") + env = encrypt.get("env") + if not isinstance(asset, str) or not ASSET_RE.fullmatch(asset): + raise ManifestError("encrypt.asset must be named like server.sops.env") if not isinstance(keys, list) or not keys: raise ManifestError("keys must be a non-empty list") if not isinstance(env, dict) or not env: @@ -86,7 +80,7 @@ def load_manifest(path): raise ManifestError(f"invalid output env name: {output_name!r}") if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): raise ManifestError(f"invalid source key for {output_name}: {source_name!r}") - return manifest + return encrypt def dotenv_value(value): @@ -144,8 +138,7 @@ def main(argv=None): args.output.chmod(0o600) write_github_outputs( { - "release_repo": manifest.get("release_repo", ""), - "release_asset": manifest.get("release_asset", f"{args.manifest.stem}.sops.env"), + "asset": manifest["asset"], "keys": ",".join(manifest["keys"]), } ) diff --git a/.github/actions/publish-sops-env/tests/test_render_env.py b/.github/actions/encrypt-env/tests/test_render_env.py similarity index 79% rename from .github/actions/publish-sops-env/tests/test_render_env.py rename to .github/actions/encrypt-env/tests/test_render_env.py index ef99fcc..00475e0 100644 --- a/.github/actions/publish-sops-env/tests/test_render_env.py +++ b/.github/actions/encrypt-env/tests/test_render_env.py @@ -28,7 +28,9 @@ def test_quotes_shell_sensitive_values(self): def test_rejects_raw_env(self): with self.assertRaises(render_env.ManifestError): - render_env.load_manifest(self.write_manifest("raw_env: [APPS]\nenv:\n APPS: APPS\n")) + render_env.load_manifest( + self.write_manifest("encrypt:\n asset: test.sops.env\n raw_env: [APPS]\n") + ) def test_missing_source_fails(self): with self.assertRaises(render_env.ManifestError): @@ -38,11 +40,12 @@ def test_duplicate_yaml_keys_fail(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "manifest.yml" path.write_text( - "release_repo: example/secrets\n" - "keys: [master, server]\n" - "env:\n" - " TOKEN: FIRST\n" - " TOKEN: SECOND\n" + "encrypt:\n" + " asset: mainframe.sops.env\n" + " keys: [master, server]\n" + " env:\n" + " TOKEN: FIRST\n" + " TOKEN: SECOND\n" ) with self.assertRaises(render_env.ManifestError): render_env.load_manifest(path) @@ -61,11 +64,11 @@ def test_main_writes_env_and_outputs(self): env_path = root / ".env" outputs_path = root / "outputs" manifest_path.write_text( - "release_repo: example/secrets\n" - "release_asset: mainframe.sops.env\n" - "keys: [master, server]\n" - "env:\n" - " TOKEN: TOKEN\n" + "encrypt:\n" + " asset: mainframe.sops.env\n" + " keys: [master, server]\n" + " env:\n" + " TOKEN: TOKEN\n" ) old_env = os.environ.copy() os.environ.update( @@ -82,9 +85,7 @@ def test_main_writes_env_and_outputs(self): os.environ.update(old_env) self.assertEqual(result, 0) self.assertIn("TOKEN=secret\n", env_path.read_text()) - self.assertIn("release_repo=example/secrets\n", outputs_path.read_text()) - self.assertIn("release_tag=\n", outputs_path.read_text()) - self.assertIn("release_asset=mainframe.sops.env\n", outputs_path.read_text()) + self.assertIn("asset=mainframe.sops.env\n", outputs_path.read_text()) self.assertIn("keys=master,server\n", outputs_path.read_text()) diff --git a/.github/actions/load-targets/README.md b/.github/actions/load-targets/README.md new file mode 100644 index 0000000..722f295 --- /dev/null +++ b/.github/actions/load-targets/README.md @@ -0,0 +1,18 @@ +# load-targets + +Composite GitHub Action that validates YAML files in `targets/` and builds a GitHub Actions matrix from either their `encrypt` or `deploy` sections. + +## Usage + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/load-targets@main + id: targets + with: + mode: deploy # encrypt or deploy + target: all # default: all + directory: targets # default: targets +``` + +The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. When `target` is `all`, files without the requested section are skipped. A specifically selected target must contain that section. + +Every target is fully validated before its requested section is added to the matrix. Target files may use either the `.yml` or `.yaml` extension, and their basename becomes the target name. diff --git a/.github/actions/load-targets/action.yml b/.github/actions/load-targets/action.yml new file mode 100644 index 0000000..de87e15 --- /dev/null +++ b/.github/actions/load-targets/action.yml @@ -0,0 +1,35 @@ +name: Load targets +description: Validate target manifests and build an encrypt or deploy strategy matrix. +inputs: + mode: + description: Target section to load (encrypt or deploy). + required: true + target: + description: Target name to load, or all. + required: false + default: all + directory: + description: Directory containing target YAML files. + required: false + default: targets +outputs: + matrix: + description: JSON strategy matrix containing validated target data. + value: ${{ steps.load.outputs.matrix }} + count: + description: Number of targets in the matrix. + value: ${{ steps.load.outputs.count }} +runs: + using: composite + steps: + - name: Install dependencies + shell: bash + run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" + - name: Load targets + id: load + shell: bash + env: + DIRECTORY: ${{ inputs.directory }} + MODE: ${{ inputs.mode }} + TARGET: ${{ inputs.target }} + run: python3 "$GITHUB_ACTION_PATH/scripts/load-targets.py" --directory "$DIRECTORY" --mode "$MODE" --target "$TARGET" diff --git a/.github/actions/load-targets/requirements.txt b/.github/actions/load-targets/requirements.txt new file mode 100644 index 0000000..8392d54 --- /dev/null +++ b/.github/actions/load-targets/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.2 diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-targets/scripts/load-targets.py new file mode 100644 index 0000000..4d618b3 --- /dev/null +++ b/.github/actions/load-targets/scripts/load-targets.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import sys +from pathlib import Path + +import yaml + +TARGET_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") +SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") +KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +RELEASE_REF_RE = re.compile( + r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$" +) +ENV_REF_RE = re.compile( + r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+\.sops\.env$" +) +ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") + + +class TargetError(Exception): + pass + + +class UniqueKeyLoader(yaml.SafeLoader): + pass + + +def construct_mapping(loader, node, deep=False): + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise TargetError(f"duplicate YAML key: {key}") + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping, +) + + +def require_mapping(value, location): + if not isinstance(value, dict): + raise TargetError(f"{location} must be a mapping") + return value + + +def reject_unknown(mapping, allowed, location): + unknown = sorted(set(mapping) - set(allowed)) + if unknown: + raise TargetError(f"{location} contains unknown keys: {', '.join(unknown)}") + + +def require_string(mapping, key, location, default=None): + value = mapping.get(key, default) + if not isinstance(value, str) or not value: + raise TargetError(f"{location}.{key} must be a non-empty string") + return value + + +def validate_encrypt(value, location): + encrypt = require_mapping(value, location) + reject_unknown(encrypt, {"asset", "keys", "env"}, location) + asset = require_string(encrypt, "asset", location) + if not ASSET_RE.fullmatch(asset): + raise TargetError(f"{location}.asset must be named like server.sops.env") + keys = encrypt.get("keys") + if not isinstance(keys, list) or not keys: + raise TargetError(f"{location}.keys must be a non-empty array") + if any(not isinstance(key, str) or not KEY_NAME_RE.fullmatch(key) for key in keys): + raise TargetError(f"{location}.keys contains an invalid age key name") + env = require_mapping(encrypt.get("env"), f"{location}.env") + if not env: + raise TargetError(f"{location}.env must not be empty") + for output_name, source_name in env.items(): + if not isinstance(output_name, str) or not ENV_NAME_RE.fullmatch(output_name): + raise TargetError(f"{location}.env contains an invalid output name: {output_name!r}") + if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): + raise TargetError(f"{location}.env.{output_name} has an invalid source name") + return {"asset": asset} + + +def validate_credentials(value, location): + credentials = require_mapping(value, location) + reject_unknown(credentials, {"variables", "secrets"}, location) + variables = require_mapping(credentials.get("variables", {}), f"{location}.variables") + secrets = require_mapping(credentials.get("secrets"), f"{location}.secrets") + reject_unknown(variables, {"tailscale_oauth_client_id"}, f"{location}.variables") + reject_unknown( + secrets, + {"ssh_private_key", "tailscale_oauth_secret"}, + f"{location}.secrets", + ) + ssh_private_key = require_string(secrets, "ssh_private_key", f"{location}.secrets") + client_id = variables.get("tailscale_oauth_client_id", "") + client_secret = secrets.get("tailscale_oauth_secret", "") + for name, value_name in ( + ("tailscale_oauth_client_id", client_id), + ("tailscale_oauth_secret", client_secret), + ): + if not isinstance(value_name, str): + raise TargetError(f"{location}.{name} must be a string") + if bool(client_id) != bool(client_secret): + raise TargetError(f"{location} must configure both Tailscale OAuth names or neither") + return { + "ssh_private_key_secret": ssh_private_key, + "tailscale_oauth_client_id_variable": client_id, + "tailscale_oauth_secret": client_secret, + } + + +def validate_deploy(value, location): + deploy = require_mapping(value, location) + reject_unknown( + deploy, + {"flightdeck_ref", "env_ref", "extra_refs", "host", "credentials", "keep_releases"}, + location, + ) + flightdeck_ref = require_string(deploy, "flightdeck_ref", location) + env_ref = require_string(deploy, "env_ref", location) + if not RELEASE_REF_RE.fullmatch(flightdeck_ref): + raise TargetError(f"{location}.flightdeck_ref must be in owner/repo@tag format") + if not ENV_REF_RE.fullmatch(env_ref): + raise TargetError(f"{location}.env_ref must be in owner/repo@tag:asset.sops.env format") + extra_refs = deploy.get("extra_refs", []) + if not isinstance(extra_refs, list): + raise TargetError(f"{location}.extra_refs must be an array") + if any(not isinstance(ref, str) or not RELEASE_REF_RE.fullmatch(ref) for ref in extra_refs): + raise TargetError(f"{location}.extra_refs contains an invalid release ref") + keep_releases = deploy.get("keep_releases", 5) + if not isinstance(keep_releases, int) or isinstance(keep_releases, bool) or keep_releases < 1: + raise TargetError(f"{location}.keep_releases must be a positive integer") + host = require_mapping(deploy.get("host"), f"{location}.host") + reject_unknown(host, {"inventory", "user", "path", "sops_age_key_file"}, f"{location}.host") + inventory = host.get("inventory") + if not isinstance(inventory, list) or not inventory: + raise TargetError(f"{location}.host.inventory must be a non-empty array") + if any(not isinstance(item, str) or not item for item in inventory): + raise TargetError(f"{location}.host.inventory must contain non-empty strings") + user = require_string(host, "user", f"{location}.host", "root") + path = require_string(host, "path", f"{location}.host", "~/flightdeck") + sops_key_file = require_string( + host, + "sops_age_key_file", + f"{location}.host", + "~/.config/sops/age/keys.txt", + ) + result = { + "flightdeck_ref": flightdeck_ref, + "env_ref": env_ref, + "extra_refs": extra_refs, + "inventory": inventory, + "user": user, + "path": path, + "keep_releases": keep_releases, + "sops_age_key_file": sops_key_file, + } + result.update(validate_credentials(deploy.get("credentials"), f"{location}.credentials")) + return result + + +def load_target(path): + try: + value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) + except yaml.YAMLError as exc: + raise TargetError(f"{path} is not valid YAML: {exc}") from exc + target = require_mapping(value, str(path)) + reject_unknown(target, {"encrypt", "deploy"}, str(path)) + if not target: + raise TargetError(f"{path} must contain encrypt or deploy") + result = {} + if "encrypt" in target: + result["encrypt"] = validate_encrypt(target["encrypt"], f"{path}.encrypt") + if "deploy" in target: + result["deploy"] = validate_deploy(target["deploy"], f"{path}.deploy") + return result + + +def build_matrix(directory, mode, selected="all"): + if mode not in {"encrypt", "deploy"}: + raise TargetError("mode must be encrypt or deploy") + paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) + if not paths: + raise TargetError(f"no target manifests found in {directory}") + names = {path.stem for path in paths} + if selected != "all" and selected not in names: + raise TargetError(f"unknown target: {selected}") + include = [] + seen_names = set() + for path in paths: + name = path.stem + if not TARGET_NAME_RE.fullmatch(name): + raise TargetError(f"invalid target filename: {path.name}") + if name in seen_names: + raise TargetError(f"duplicate target name: {name}") + seen_names.add(name) + target = load_target(path) + if selected != "all" and name != selected: + continue + if mode not in target: + if selected != "all": + raise TargetError(f"target {name} has no {mode} section") + continue + item = {"target": name, "manifest": str(path)} + item.update(target[mode]) + include.append(item) + return {"include": include} + + +def write_github_output(name, value): + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + output.write(f"{name}={value}\n") + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--directory", type=Path, default=Path("targets")) + parser.add_argument("--mode", required=True) + parser.add_argument("--target", default="all") + args = parser.parse_args(argv) + try: + matrix = build_matrix(args.directory, args.mode, args.target) + encoded = json.dumps(matrix, separators=(",", ":")) + write_github_output("matrix", encoded) + write_github_output("count", len(matrix["include"])) + except TargetError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py new file mode 100644 index 0000000..eeab122 --- /dev/null +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -0,0 +1,95 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-targets.py" +SPEC = importlib.util.spec_from_file_location("load_targets", MODULE_PATH) +load_targets = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(load_targets) + + +TARGET = """\ +encrypt: + asset: hawkeye.sops.env + keys: [hawkeye] + env: + APPS: RUBYKATZEN_COM_APPS +deploy: + flightdeck_ref: rubykatzen/flightdeck@v1.2.3 + env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env + extra_refs: + - rubykatzen/apps@v3.0.0:flightdeck-extra.zip + host: + inventory: [100.75.50.2] + user: rubykatzen-com + path: ~/flightdeck + sops_age_key_file: ~/.config/sops/age/keys.txt + credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET +""" + + +class LoadTargetsTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.directory = Path(self.temporary_directory.name) + (self.directory / "hawkeye.yml").write_text(TARGET) + + def test_builds_encrypt_matrix(self): + matrix = load_targets.build_matrix(self.directory, "encrypt") + self.assertEqual( + matrix, + { + "include": [ + { + "target": "hawkeye", + "manifest": str(self.directory / "hawkeye.yml"), + "asset": "hawkeye.sops.env", + } + ] + }, + ) + + def test_builds_deploy_matrix(self): + item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + self.assertEqual(item["inventory"], ["100.75.50.2"]) + self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") + self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) + self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") + self.assertNotIn("RUBYKATZEN_COM_APPS", item) + + def test_filters_selected_target(self): + (self.directory / "other.yml").write_text(TARGET.replace("hawkeye.sops.env", "other.sops.env")) + matrix = load_targets.build_matrix(self.directory, "encrypt", "hawkeye") + self.assertEqual([item["target"] for item in matrix["include"]], ["hawkeye"]) + + def test_rejects_unknown_target(self): + with self.assertRaisesRegex(load_targets.TargetError, "unknown target"): + load_targets.build_matrix(self.directory, "deploy", "missing") + + def test_rejects_missing_selected_section(self): + (self.directory / "encrypt-only.yml").write_text(TARGET.split("deploy:\n", maxsplit=1)[0]) + with self.assertRaisesRegex(load_targets.TargetError, "has no deploy section"): + load_targets.build_matrix(self.directory, "deploy", "encrypt-only") + + def test_rejects_string_inventory(self): + path = self.directory / "hawkeye.yml" + path.write_text(TARGET.replace("inventory: [100.75.50.2]", 'inventory: "100.75.50.2,"')) + with self.assertRaisesRegex(load_targets.TargetError, "inventory must be a non-empty array"): + load_targets.build_matrix(self.directory, "deploy") + + def test_rejects_duplicate_keys(self): + path = self.directory / "hawkeye.yml" + path.write_text(TARGET.replace(" asset: hawkeye.sops.env", " asset: one.sops.env\n asset: two.sops.env")) + with self.assertRaisesRegex(load_targets.TargetError, "duplicate YAML key"): + load_targets.build_matrix(self.directory, "encrypt") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/publish-sops-env/README.md b/.github/actions/publish-sops-env/README.md deleted file mode 100644 index ea4b28a..0000000 --- a/.github/actions/publish-sops-env/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# publish-sops-env - -Composite GitHub Action that renders an env manifest from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` to an existing GitHub Release. - -The release must exist before this action runs. - -## Usage - -```yaml -- uses: rubykatzen/flightdeck/.github/actions/publish-sops-env@main - with: - manifest: projects/flightdeck/mainframe.yml # required - keys-directory: keys # default: keys - release-tag: ${{ needs.release.outputs.tag }} # required, must already exist - release-repo: "" # default: current repository - asset-name: "" # default: manifest release_asset or .sops.env - token: ${{ secrets.GITHUB_TOKEN }} # required - env: - GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} - GITHUB_VARS_JSON: ${{ toJson(vars) }} -``` - -The calling job requires: - -```yaml -permissions: - contents: write -``` - -## Manifest - -```yaml -release_asset: flightdeck--mainframe.sops.env - -keys: - - mainframe - -env: - APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name - APPS_TIMEZONE: APPS_TIMEZONE - APPS: APPS_MAINFRAME -``` - -For each name in `keys`, the action loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 7c470a2..4a26fbf 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -3,7 +3,7 @@ on: workflow_call: inputs: inventory: - description: Ansible inventory, e.g. a comma-separated Tailscale host list. + description: JSON array of inventory hosts. type: string required: true user: @@ -11,7 +11,7 @@ on: type: string default: root app-ref: - description: "Release ref of the Flightdeck bundle to deploy (e.g. a version tag, or 'latest'). Independent of the @tag on the uses: line, which only controls which ref runs the playbook mechanism." + description: Full release ref of the Flightdeck bundle to deploy, in owner/repo@tag format. type: string required: true env-ref: @@ -19,9 +19,9 @@ on: type: string required: true extra-refs: - description: Comma-separated release refs for optional extra app bundles. + description: JSON array of release refs for optional extra app bundles. type: string - default: "" + default: "[]" path: description: Base path on the target host for releases, shared files, and the current symlink. type: string @@ -77,9 +77,10 @@ jobs: id: vars shell: bash env: - APP_REF: rubykatzen/flightdeck@${{ inputs.app-ref }} + APP_REF: ${{ inputs.app-ref }} ENV_REF: ${{ inputs.env-ref }} EXTRA_REFS: ${{ inputs.extra-refs }} + INVENTORY: ${{ inputs.inventory }} DEPLOY_PATH: ${{ inputs.path }} KEEP_RELEASES: ${{ inputs.keep-releases }} SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} @@ -94,11 +95,12 @@ jobs: sops_key_file="$SOPS_KEY_FILE" if [ -z "$sops_key_file" ]; then sops_key_file="$home_dir/.config/sops/age/keys.txt" + else + sops_key_file="${sops_key_file/#\~/$home_dir}" fi - extra_refs_json="[]" - if [ -n "$EXTRA_REFS" ]; then - extra_refs_json="$(printf '%s' "$EXTRA_REFS" | jq -R 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')" - fi + inventory_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("inventory must be a non-empty string array") end' <<< "$INVENTORY")" + extra_refs_json="$(jq -ce 'if type == "array" and all(.[]; type == "string" and length > 0) then . else error("extra-refs must be a string array") end' <<< "$EXTRA_REFS")" + inventory="$(jq -r 'join(",") + ","' <<< "$inventory_json")" json="$(jq -n \ --arg app_ref "$APP_REF" \ --arg env_ref "$ENV_REF" \ @@ -108,12 +110,13 @@ jobs: --arg sops_key_file "$sops_key_file" \ '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_extra_refs: $extra_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" echo "json=$json" >> "$GITHUB_OUTPUT" + echo "inventory=$inventory" >> "$GITHUB_OUTPUT" - name: Run playbook shell: bash env: ANSIBLE_HOST_KEY_CHECKING: "false" run: | ansible-playbook ansible/deploy.yml \ - -i "${{ inputs.inventory }}" \ + -i "${{ steps.vars.outputs.inventory }}" \ -u "${{ inputs.user }}" \ -e "${{ steps.vars.outputs.json }}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 038de40..c1302fd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,26 +4,45 @@ on: inputs: target: description: Deploy target to redeploy. Use "all" to redeploy every target. - type: choice - options: - - all - - hawkeye + type: string default: all workflow_run: workflows: [Release] types: [completed] jobs: - hawkeye: + targets: if: >- - (github.event_name == 'workflow_dispatch' && (inputs.target == 'all' || inputs.target == 'hawkeye')) || + github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') - uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@main + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + count: ${{ steps.targets.outputs.count }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - uses: $/.github/actions/load-targets + id: targets + with: + mode: deploy + target: ${{ inputs.target || 'all' }} + deploy: + needs: targets + if: needs.targets.outputs.count != '0' + strategy: + matrix: ${{ fromJson(needs.targets.outputs.matrix) }} + uses: $/.github/workflows/deploy-shared.yml with: - inventory: "100.75.50.2," - user: rubykatzen-com - app-ref: latest - env-ref: "${{ github.repository }}@latest:hawkeye.sops.env" - tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} + inventory: ${{ toJson(matrix.inventory) }} + user: ${{ matrix.user }} + app-ref: ${{ matrix.flightdeck_ref }} + env-ref: ${{ matrix.env_ref }} + extra-refs: ${{ toJson(matrix.extra_refs) }} + path: ${{ matrix.path }} + keep-releases: ${{ matrix.keep_releases }} + sops-age-key-file: ${{ matrix.sops_age_key_file }} + tailscale-oauth-client-id: ${{ vars[matrix.tailscale_oauth_client_id_variable] }} secrets: - ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} - tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + tailscale-oauth-secret: ${{ secrets[matrix.tailscale_oauth_secret] }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 379b66b..a42bba8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,6 @@ jobs: id: release with: token: ${{ secrets.RELEASE_TOKEN }} - upload: needs: release if: needs.release.outputs.release_created == 'true' @@ -44,15 +43,34 @@ jobs: restart.sh up.sh README.md - hawkeye-env: + targets: needs: release if: needs.release.outputs.release_created == 'true' runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + count: ${{ steps.targets.outputs.count }} steps: - uses: actions/checkout@v7 - - uses: $/.github/actions/publish-sops-env with: - manifest: targets/hawkeye.yml + ref: ${{ needs.release.outputs.tag_name }} + - uses: $/.github/actions/load-targets + id: targets + with: + mode: encrypt + encrypt: + needs: [release, targets] + if: needs.targets.outputs.count != '0' + strategy: + matrix: ${{ fromJson(needs.targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.release.outputs.tag_name }} + - uses: $/.github/actions/encrypt-env + with: + manifest: ${{ matrix.manifest }} release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} env: diff --git a/AGENTS.md b/AGENTS.md index 1cd93ec..fd09224 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,7 +409,7 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release Deployment helpers live in this repository: - `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` -- `.github/actions/publish-sops-env/` is a local composite action for rendering env manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset +- `.github/actions/encrypt-env/` is a local composite action for rendering target env manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. diff --git a/README.md b/README.md index 54aaa17..a11101f 100644 --- a/README.md +++ b/README.md @@ -174,11 +174,13 @@ flightdeck/ ├── .github/ │ ├── actions/ │ │ ├── build-bundle/ # Build and upload a release bundle -│ │ └── publish-sops-env/ # Encrypt env manifest and upload to GitHub Release +│ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release +│ │ └── load-targets/ # Validate targets and build workflow matrices │ └── workflows/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ +├── targets/ # Independent encryption and deployment targets ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template │ @@ -438,22 +440,57 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides two composite actions under `.github/actions/` (`build-bundle` and `publish-sops-env`) and one reusable workflow, `deploy-shared.yml`. +This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-targets`) and one reusable workflow, `deploy-shared.yml`. --- -### `publish-sops-env` +### Targets -Renders an env manifest from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` to an existing GitHub Release. Release creation remains the calling workflow's responsibility. +Each file in `targets/` describes one logical target. Its optional root sections are independent: an encryption-only repository can omit `deploy`, while an inventory-only repository can omit `encrypt`. A target containing both ties the encrypted env asset and deployment inventory together by name. ```yaml -- uses: rubykatzen/flightdeck/.github/actions/publish-sops-env@main +encrypt: + asset: mainframe.sops.env + keys: + - mainframe + env: + APPS: MAINFRAME_APPS + APPS_DOMAIN: MAINFRAME_DOMAIN +deploy: + flightdeck_ref: rubykatzen/flightdeck@latest + env_ref: owner/config@latest:mainframe.sops.env + extra_refs: + - owner/extra-apps@latest + host: + inventory: + - 100.64.0.1 + - 100.64.0.2 + user: deploy + path: ~/flightdeck + sops_age_key_file: ~/.config/sops/age/keys.txt + credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET +``` + +Credential fields contain GitHub Variable/Secret names, never credential values. `extra_refs` and `host.inventory` are YAML arrays. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. + +--- + +### `encrypt-env` + +Renders the `encrypt` section of a target from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` to an existing GitHub Release. Release creation remains the calling workflow's responsibility. + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: projects/flightdeck/mainframe.yml # required + manifest: targets/mainframe.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository - asset-name: "" # default: manifest release_asset or .sops.env token: ${{ secrets.GITHUB_TOKEN }} # required env: GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} @@ -465,14 +502,13 @@ Requires `contents: write` permission on the calling job. **Manifest format:** ```yaml -release_asset: flightdeck--mainframe.sops.env - -keys: - - mainframe - -env: - APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name - APPS: APPS_AGATHA +encrypt: + asset: mainframe.sops.env + keys: + - mainframe + env: + APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name + APPS: APPS_MAINFRAME ``` Secrets take precedence over Variables when both contain the same source key. Every source key must exist or the action fails. @@ -513,11 +549,11 @@ jobs: deploy: uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v1.2.3 with: - inventory: 100.64.0.1,100.64.0.2 # required + inventory: '["100.64.0.1", "100.64.0.2"]' # required JSON array user: root # default: root - app-ref: latest # required (e.g. a version tag, or "latest") + app-ref: rubykatzen/flightdeck@latest # required full release ref env-ref: "${{ github.repository }}@latest:.sops.env" # required - # extra-refs: owner/repo@latest # optional, comma-separated, default: none + # extra-refs: '["owner/repo@latest"]' # optional JSON array, default: [] # path: ~/flightdeck # optional, default shown # keep-releases: 5 # optional, default shown # sops-age-key-file: /home/deploy/.config/sops/age/keys.txt # optional, default: ~/.config/sops/age/keys.txt for `user` @@ -528,7 +564,7 @@ jobs: tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # optional, required only if tailscale-oauth-client-id is set ``` -The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required — it's the release bundle the playbook downloads and deploys, and doesn't have to match the pin (e.g. pin to a stable mechanism version but pass `app-ref: latest` to always deploy the newest release). +The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required - it is the full release ref for the bundle the playbook downloads and deploys, and does not have to match the workflow pin. ## 📝 License diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 74c2ac1..52039c7 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,16 +1,31 @@ -release_asset: hawkeye.sops.env - -keys: - - hawkeye - -env: - APPS: RUBYKATZEN_COM_APPS - APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN - APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL - APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER - APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN - APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD - APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 - APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE - TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT - TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT +encrypt: + asset: hawkeye.sops.env + keys: + - hawkeye + env: + APPS: RUBYKATZEN_COM_APPS + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN + APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL + APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER + APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN + APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD + APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 + APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE + TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT + TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT +deploy: + flightdeck_ref: rubykatzen/flightdeck@latest + env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env + extra_refs: [] + host: + inventory: + - 100.75.50.2 + user: rubykatzen-com + path: ~/flightdeck + sops_age_key_file: ~/.config/sops/age/keys.txt + credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET From a1db561d30fde0a8d46e180e711d98780ce40aba Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 02:07:39 +0200 Subject: [PATCH 21/35] feat: define target apps in encryption config --- .github/actions/encrypt-env/README.md | 6 ++-- .../actions/encrypt-env/scripts/render-env.py | 15 ++++++-- .../encrypt-env/tests/test_render_env.py | 36 +++++++++++++++++-- .../load-targets/scripts/load-targets.py | 12 ++++++- .../load-targets/tests/test_load_targets.py | 17 +++++++-- README.md | 10 ++++-- targets/hawkeye.yml | 4 ++- 7 files changed, 86 insertions(+), 14 deletions(-) diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md index f637368..afa3698 100644 --- a/.github/actions/encrypt-env/README.md +++ b/.github/actions/encrypt-env/README.md @@ -33,10 +33,12 @@ encrypt: asset: mainframe.sops.env keys: - mainframe + apps: + - traefik + - rybbit env: APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name APPS_TIMEZONE: APPS_TIMEZONE - APPS: APPS_MAINFRAME ``` -For each name in `encrypt.keys`, the action loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. +The action renders `encrypt.apps` as the comma-separated `APPS` dotenv value. For each name in `encrypt.keys`, it loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. diff --git a/.github/actions/encrypt-env/scripts/render-env.py b/.github/actions/encrypt-env/scripts/render-env.py index 2dd7f0b..f629c34 100644 --- a/.github/actions/encrypt-env/scripts/render-env.py +++ b/.github/actions/encrypt-env/scripts/render-env.py @@ -12,6 +12,7 @@ ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +APP_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") @@ -60,21 +61,31 @@ def load_manifest(path): encrypt = manifest.get("encrypt") if not isinstance(encrypt, dict): raise ManifestError(f"{path} must contain an encrypt mapping") - unknown = sorted(set(encrypt) - {"asset", "keys", "env"}) + unknown = sorted(set(encrypt) - {"asset", "keys", "apps", "env"}) if unknown: raise ManifestError("encrypt contains unknown keys: " + ", ".join(unknown)) asset = encrypt.get("asset") keys = encrypt.get("keys") + apps = encrypt.get("apps") env = encrypt.get("env") if not isinstance(asset, str) or not ASSET_RE.fullmatch(asset): raise ManifestError("encrypt.asset must be named like server.sops.env") if not isinstance(keys, list) or not keys: raise ManifestError("keys must be a non-empty list") + if not isinstance(apps, list) or not apps: + raise ManifestError("apps must be a non-empty list") if not isinstance(env, dict) or not env: raise ManifestError("env must be a non-empty mapping") for key in keys: if not isinstance(key, str) or not KEY_NAME_RE.fullmatch(key): raise ManifestError(f"invalid key name: {key!r}") + for app in apps: + if not isinstance(app, str) or not APP_NAME_RE.fullmatch(app): + raise ManifestError(f"invalid app name: {app!r}") + if len(apps) != len(set(apps)): + raise ManifestError("apps contains duplicate app names") + if "APPS" in env: + raise ManifestError("APPS must be configured through encrypt.apps") for output_name, source_name in env.items(): if not isinstance(output_name, str) or not ENV_NAME_RE.fullmatch(output_name): raise ManifestError(f"invalid output env name: {output_name!r}") @@ -103,7 +114,7 @@ def resolve_value(source_name, secrets, variables): def render_env(manifest, secrets, variables): - lines = [] + lines = [f"APPS={','.join(manifest['apps'])}"] missing = [] for output_name, source_name in manifest["env"].items(): value = resolve_value(source_name, secrets, variables) diff --git a/.github/actions/encrypt-env/tests/test_render_env.py b/.github/actions/encrypt-env/tests/test_render_env.py index 00475e0..062fad0 100644 --- a/.github/actions/encrypt-env/tests/test_render_env.py +++ b/.github/actions/encrypt-env/tests/test_render_env.py @@ -13,7 +13,7 @@ class RenderEnvTest(unittest.TestCase): def test_render_prefers_secrets_over_variables(self): - manifest = {"env": {"DOMAIN": "DOMAIN", "TIMEZONE": "TIMEZONE"}} + manifest = {"apps": ["traefik", "rybbit"], "env": {"DOMAIN": "DOMAIN", "TIMEZONE": "TIMEZONE"}} output = render_env.render_env( manifest, {"DOMAIN": "secret.example"}, @@ -21,9 +21,12 @@ def test_render_prefers_secrets_over_variables(self): ) self.assertIn("DOMAIN=secret.example\n", output) self.assertIn("TIMEZONE=Europe/Berlin\n", output) + self.assertIn("APPS=traefik,rybbit\n", output) def test_quotes_shell_sensitive_values(self): - output = render_env.render_env({"env": {"TOKEN": "TOKEN"}}, {"TOKEN": "hello world"}, {}) + output = render_env.render_env( + {"apps": ["traefik"], "env": {"TOKEN": "TOKEN"}}, {"TOKEN": "hello world"}, {} + ) self.assertIn("TOKEN='hello world'\n", output) def test_rejects_raw_env(self): @@ -32,9 +35,33 @@ def test_rejects_raw_env(self): self.write_manifest("encrypt:\n asset: test.sops.env\n raw_env: [APPS]\n") ) + def test_rejects_apps_in_env(self): + manifest = ( + "encrypt:\n" + " asset: test.sops.env\n" + " keys: [test]\n" + " apps: [traefik]\n" + " env:\n" + " APPS: TEST_APPS\n" + ) + with self.assertRaisesRegex(render_env.ManifestError, "must be configured through"): + render_env.load_manifest(self.write_manifest(manifest)) + + def test_rejects_duplicate_apps(self): + manifest = ( + "encrypt:\n" + " asset: test.sops.env\n" + " keys: [test]\n" + " apps: [traefik, traefik]\n" + " env:\n" + " TOKEN: TOKEN\n" + ) + with self.assertRaisesRegex(render_env.ManifestError, "duplicate app names"): + render_env.load_manifest(self.write_manifest(manifest)) + def test_missing_source_fails(self): with self.assertRaises(render_env.ManifestError): - render_env.render_env({"env": {"TOKEN": "TOKEN"}}, {}, {}) + render_env.render_env({"apps": ["traefik"], "env": {"TOKEN": "TOKEN"}}, {}, {}) def test_duplicate_yaml_keys_fail(self): with tempfile.TemporaryDirectory() as directory: @@ -43,6 +70,7 @@ def test_duplicate_yaml_keys_fail(self): "encrypt:\n" " asset: mainframe.sops.env\n" " keys: [master, server]\n" + " apps: [traefik, rybbit]\n" " env:\n" " TOKEN: FIRST\n" " TOKEN: SECOND\n" @@ -67,6 +95,7 @@ def test_main_writes_env_and_outputs(self): "encrypt:\n" " asset: mainframe.sops.env\n" " keys: [master, server]\n" + " apps: [traefik, rybbit]\n" " env:\n" " TOKEN: TOKEN\n" ) @@ -85,6 +114,7 @@ def test_main_writes_env_and_outputs(self): os.environ.update(old_env) self.assertEqual(result, 0) self.assertIn("TOKEN=secret\n", env_path.read_text()) + self.assertIn("APPS=traefik,rybbit\n", env_path.read_text()) self.assertIn("asset=mainframe.sops.env\n", outputs_path.read_text()) self.assertIn("keys=master,server\n", outputs_path.read_text()) diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-targets/scripts/load-targets.py index 4d618b3..53d5a10 100644 --- a/.github/actions/load-targets/scripts/load-targets.py +++ b/.github/actions/load-targets/scripts/load-targets.py @@ -12,6 +12,7 @@ ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +APP_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") RELEASE_REF_RE = re.compile( r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$" ) @@ -66,7 +67,7 @@ def require_string(mapping, key, location, default=None): def validate_encrypt(value, location): encrypt = require_mapping(value, location) - reject_unknown(encrypt, {"asset", "keys", "env"}, location) + reject_unknown(encrypt, {"asset", "keys", "apps", "env"}, location) asset = require_string(encrypt, "asset", location) if not ASSET_RE.fullmatch(asset): raise TargetError(f"{location}.asset must be named like server.sops.env") @@ -75,6 +76,13 @@ def validate_encrypt(value, location): raise TargetError(f"{location}.keys must be a non-empty array") if any(not isinstance(key, str) or not KEY_NAME_RE.fullmatch(key) for key in keys): raise TargetError(f"{location}.keys contains an invalid age key name") + apps = encrypt.get("apps") + if not isinstance(apps, list) or not apps: + raise TargetError(f"{location}.apps must be a non-empty array") + if any(not isinstance(app, str) or not APP_NAME_RE.fullmatch(app) for app in apps): + raise TargetError(f"{location}.apps contains an invalid app name") + if len(apps) != len(set(apps)): + raise TargetError(f"{location}.apps contains duplicate app names") env = require_mapping(encrypt.get("env"), f"{location}.env") if not env: raise TargetError(f"{location}.env must not be empty") @@ -83,6 +91,8 @@ def validate_encrypt(value, location): raise TargetError(f"{location}.env contains an invalid output name: {output_name!r}") if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): raise TargetError(f"{location}.env.{output_name} has an invalid source name") + if "APPS" in env: + raise TargetError(f"{location}.env.APPS must be configured through {location}.apps") return {"asset": asset} diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py index eeab122..930e67d 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -13,8 +13,9 @@ encrypt: asset: hawkeye.sops.env keys: [hawkeye] + apps: [traefik, rybbit] env: - APPS: RUBYKATZEN_COM_APPS + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN deploy: flightdeck_ref: rubykatzen/flightdeck@v1.2.3 env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env @@ -62,7 +63,7 @@ def test_builds_deploy_matrix(self): self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") - self.assertNotIn("RUBYKATZEN_COM_APPS", item) + self.assertNotIn("apps", item) def test_filters_selected_target(self): (self.directory / "other.yml").write_text(TARGET.replace("hawkeye.sops.env", "other.sops.env")) @@ -84,6 +85,18 @@ def test_rejects_string_inventory(self): with self.assertRaisesRegex(load_targets.TargetError, "inventory must be a non-empty array"): load_targets.build_matrix(self.directory, "deploy") + def test_rejects_apps_in_env(self): + path = self.directory / "hawkeye.yml" + path.write_text(TARGET.replace(" APPS_DOMAIN:", " APPS: RUBYKATZEN_COM_APPS\n APPS_DOMAIN:")) + with self.assertRaisesRegex(load_targets.TargetError, "must be configured through"): + load_targets.build_matrix(self.directory, "encrypt") + + def test_rejects_duplicate_apps(self): + path = self.directory / "hawkeye.yml" + path.write_text(TARGET.replace("apps: [traefik, rybbit]", "apps: [traefik, rybbit, traefik]")) + with self.assertRaisesRegex(load_targets.TargetError, "duplicate app names"): + load_targets.build_matrix(self.directory, "encrypt") + def test_rejects_duplicate_keys(self): path = self.directory / "hawkeye.yml" path.write_text(TARGET.replace(" asset: hawkeye.sops.env", " asset: one.sops.env\n asset: two.sops.env")) diff --git a/README.md b/README.md index a11101f..0bf5295 100644 --- a/README.md +++ b/README.md @@ -453,8 +453,10 @@ encrypt: asset: mainframe.sops.env keys: - mainframe + apps: + - traefik + - rybbit env: - APPS: MAINFRAME_APPS APPS_DOMAIN: MAINFRAME_DOMAIN deploy: flightdeck_ref: rubykatzen/flightdeck@latest @@ -476,7 +478,7 @@ deploy: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `extra_refs` and `host.inventory` are YAML arrays. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `host.inventory` are YAML arrays. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. --- @@ -506,9 +508,11 @@ encrypt: asset: mainframe.sops.env keys: - mainframe + apps: + - traefik + - rybbit env: APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name - APPS: APPS_MAINFRAME ``` Secrets take precedence over Variables when both contain the same source key. Every source key must exist or the action fails. diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 52039c7..7dfebf7 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -2,8 +2,10 @@ encrypt: asset: hawkeye.sops.env keys: - hawkeye + apps: + - traefik + - rybbit env: - APPS: RUBYKATZEN_COM_APPS APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER From ae0caa11559fe999908d6ead2b190d53e5225043 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 02:14:04 +0200 Subject: [PATCH 22/35] refactor: use default target paths --- .../load-targets/tests/test_load_targets.py | 24 +++++++++++++++++++ targets/hawkeye.yml | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py index 930e67d..d1a5906 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -62,9 +62,33 @@ def test_builds_deploy_matrix(self): self.assertEqual(item["inventory"], ["100.75.50.2"]) self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) + self.assertEqual(item["path"], "~/flightdeck") + self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") self.assertNotIn("apps", item) + def test_allows_host_default_overrides(self): + path = self.directory / "hawkeye.yml" + path.write_text( + TARGET.replace(" path: ~/flightdeck\n", "").replace( + " sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" + ) + ) + item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + self.assertEqual(item["path"], "~/flightdeck") + self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") + + path.write_text( + TARGET.replace(" path: ~/flightdeck", " path: ~/custom") + .replace( + " sops_age_key_file: ~/.config/sops/age/keys.txt", + " sops_age_key_file: ~/.config/sops/age/custom.txt", + ) + ) + item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + self.assertEqual(item["path"], "~/custom") + self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/custom.txt") + def test_filters_selected_target(self): (self.directory / "other.yml").write_text(TARGET.replace("hawkeye.sops.env", "other.sops.env")) matrix = load_targets.build_matrix(self.directory, "encrypt", "hawkeye") diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 7dfebf7..56e8fe6 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -23,8 +23,6 @@ deploy: inventory: - 100.75.50.2 user: rubykatzen-com - path: ~/flightdeck - sops_age_key_file: ~/.config/sops/age/keys.txt credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID From c57535cdcd6f1f736ed63652f6337b9b6d8161a5 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 02:19:08 +0200 Subject: [PATCH 23/35] refactor: hide ansible inventory from deploy interface --- .../actions/load-targets/scripts/load-targets.py | 14 +++++++------- .../load-targets/tests/test_load_targets.py | 10 +++++----- .github/workflows/deploy-shared.yml | 14 +++++++------- .github/workflows/deploy.yml | 2 +- README.md | 12 ++++++------ targets/hawkeye.yml | 2 +- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-targets/scripts/load-targets.py index 53d5a10..35ac35c 100644 --- a/.github/actions/load-targets/scripts/load-targets.py +++ b/.github/actions/load-targets/scripts/load-targets.py @@ -147,12 +147,12 @@ def validate_deploy(value, location): if not isinstance(keep_releases, int) or isinstance(keep_releases, bool) or keep_releases < 1: raise TargetError(f"{location}.keep_releases must be a positive integer") host = require_mapping(deploy.get("host"), f"{location}.host") - reject_unknown(host, {"inventory", "user", "path", "sops_age_key_file"}, f"{location}.host") - inventory = host.get("inventory") - if not isinstance(inventory, list) or not inventory: - raise TargetError(f"{location}.host.inventory must be a non-empty array") - if any(not isinstance(item, str) or not item for item in inventory): - raise TargetError(f"{location}.host.inventory must contain non-empty strings") + reject_unknown(host, {"addresses", "user", "path", "sops_age_key_file"}, f"{location}.host") + addresses = host.get("addresses") + if not isinstance(addresses, list) or not addresses: + raise TargetError(f"{location}.host.addresses must be a non-empty array") + if any(not isinstance(item, str) or not item for item in addresses): + raise TargetError(f"{location}.host.addresses must contain non-empty strings") user = require_string(host, "user", f"{location}.host", "root") path = require_string(host, "path", f"{location}.host", "~/flightdeck") sops_key_file = require_string( @@ -165,7 +165,7 @@ def validate_deploy(value, location): "flightdeck_ref": flightdeck_ref, "env_ref": env_ref, "extra_refs": extra_refs, - "inventory": inventory, + "hosts": addresses, "user": user, "path": path, "keep_releases": keep_releases, diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py index d1a5906..718b20e 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -22,7 +22,7 @@ extra_refs: - rubykatzen/apps@v3.0.0:flightdeck-extra.zip host: - inventory: [100.75.50.2] + addresses: [100.75.50.2] user: rubykatzen-com path: ~/flightdeck sops_age_key_file: ~/.config/sops/age/keys.txt @@ -59,7 +59,7 @@ def test_builds_encrypt_matrix(self): def test_builds_deploy_matrix(self): item = load_targets.build_matrix(self.directory, "deploy")["include"][0] - self.assertEqual(item["inventory"], ["100.75.50.2"]) + self.assertEqual(item["hosts"], ["100.75.50.2"]) self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) self.assertEqual(item["path"], "~/flightdeck") @@ -103,10 +103,10 @@ def test_rejects_missing_selected_section(self): with self.assertRaisesRegex(load_targets.TargetError, "has no deploy section"): load_targets.build_matrix(self.directory, "deploy", "encrypt-only") - def test_rejects_string_inventory(self): + def test_rejects_string_addresses(self): path = self.directory / "hawkeye.yml" - path.write_text(TARGET.replace("inventory: [100.75.50.2]", 'inventory: "100.75.50.2,"')) - with self.assertRaisesRegex(load_targets.TargetError, "inventory must be a non-empty array"): + path.write_text(TARGET.replace("addresses: [100.75.50.2]", 'addresses: "100.75.50.2"')) + with self.assertRaisesRegex(load_targets.TargetError, "addresses must be a non-empty array"): load_targets.build_matrix(self.directory, "deploy") def test_rejects_apps_in_env(self): diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 4a26fbf..58a457c 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -2,8 +2,8 @@ name: Deploy (shared) on: workflow_call: inputs: - inventory: - description: JSON array of inventory hosts. + hosts: + description: JSON array of host addresses to deploy to. type: string required: true user: @@ -35,7 +35,7 @@ on: type: string default: "" tailscale-oauth-client-id: - description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the inventory hosts). + description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). type: string default: "" tailscale-tags: @@ -44,7 +44,7 @@ on: default: tag:ci secrets: ssh-private-key: - description: SSH private key used to connect to the inventory hosts. + description: SSH private key used to connect to the hosts. required: true tailscale-oauth-secret: description: Tailscale OAuth client secret used to join the tailnet. Required only when tailscale-oauth-client-id is set. @@ -80,7 +80,7 @@ jobs: APP_REF: ${{ inputs.app-ref }} ENV_REF: ${{ inputs.env-ref }} EXTRA_REFS: ${{ inputs.extra-refs }} - INVENTORY: ${{ inputs.inventory }} + HOSTS: ${{ inputs.hosts }} DEPLOY_PATH: ${{ inputs.path }} KEEP_RELEASES: ${{ inputs.keep-releases }} SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} @@ -98,9 +98,9 @@ jobs: else sops_key_file="${sops_key_file/#\~/$home_dir}" fi - inventory_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("inventory must be a non-empty string array") end' <<< "$INVENTORY")" + hosts_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("hosts must be a non-empty string array") end' <<< "$HOSTS")" extra_refs_json="$(jq -ce 'if type == "array" and all(.[]; type == "string" and length > 0) then . else error("extra-refs must be a string array") end' <<< "$EXTRA_REFS")" - inventory="$(jq -r 'join(",") + ","' <<< "$inventory_json")" + inventory="$(jq -r 'join(",") + ","' <<< "$hosts_json")" json="$(jq -n \ --arg app_ref "$APP_REF" \ --arg env_ref "$ENV_REF" \ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c1302fd..95765aa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -34,7 +34,7 @@ jobs: matrix: ${{ fromJson(needs.targets.outputs.matrix) }} uses: $/.github/workflows/deploy-shared.yml with: - inventory: ${{ toJson(matrix.inventory) }} + hosts: ${{ toJson(matrix.hosts) }} user: ${{ matrix.user }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} diff --git a/README.md b/README.md index 0bf5295..95cba1a 100644 --- a/README.md +++ b/README.md @@ -446,7 +446,7 @@ This repository provides three composite actions under `.github/actions/` (`buil ### Targets -Each file in `targets/` describes one logical target. Its optional root sections are independent: an encryption-only repository can omit `deploy`, while an inventory-only repository can omit `encrypt`. A target containing both ties the encrypted env asset and deployment inventory together by name. +Each file in `targets/` describes one logical target. Its optional root sections are independent: an encryption-only repository can omit `deploy`, while a deployment-only repository can omit `encrypt`. A target containing both ties the encrypted env asset and deployment hosts together by name. ```yaml encrypt: @@ -464,7 +464,7 @@ deploy: extra_refs: - owner/extra-apps@latest host: - inventory: + addresses: - 100.64.0.1 - 100.64.0.2 user: deploy @@ -478,7 +478,7 @@ deploy: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `host.inventory` are YAML arrays. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `host.addresses` are YAML arrays. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. --- @@ -542,18 +542,18 @@ Requires `contents: write` permission on the calling job. `flightdeck-extra.zip` ### `deploy-shared.yml` -Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied inventory. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `env-ref` typically references that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. +Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied hosts. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `env-ref` typically references that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. The interface is plain deploy vocabulary, not Ansible's — callers never see `flightdeck_*` variable names or hand-write `-e` JSON; the workflow builds that internally. -Tailscale is optional, not a dependency of this workflow: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the inventory hosts, or reaches them some other way. +Tailscale is optional, not a dependency of this workflow: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the hosts, or reaches them some other way. ```yaml jobs: deploy: uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v1.2.3 with: - inventory: '["100.64.0.1", "100.64.0.2"]' # required JSON array + hosts: '["100.64.0.1", "100.64.0.2"]' # required JSON array user: root # default: root app-ref: rubykatzen/flightdeck@latest # required full release ref env-ref: "${{ github.repository }}@latest:.sops.env" # required diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 56e8fe6..4fb9219 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -20,7 +20,7 @@ deploy: env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env extra_refs: [] host: - inventory: + addresses: - 100.75.50.2 user: rubykatzen-com credentials: From 9463898de18cfa7721cfb406b4f7e70edf561345 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 02:22:32 +0200 Subject: [PATCH 24/35] refactor: describe deploy hosts as ssh destinations --- .../load-targets/scripts/load-targets.py | 41 ++++++++++++------- .../load-targets/tests/test_load_targets.py | 37 ++++++++++------- .github/workflows/deploy-shared.yml | 39 ++++++------------ .github/workflows/deploy.yml | 1 - README.md | 17 ++++---- ansible/deploy.yml | 7 ++++ targets/hawkeye.yml | 6 +-- 7 files changed, 79 insertions(+), 69 deletions(-) diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-targets/scripts/load-targets.py index 35ac35c..16059c0 100644 --- a/.github/actions/load-targets/scripts/load-targets.py +++ b/.github/actions/load-targets/scripts/load-targets.py @@ -13,6 +13,9 @@ SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") APP_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +SSH_DESTINATION_RE = re.compile( + r"^(?P[a-z_][a-z0-9_-]*)@(?P[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?)$" +) RELEASE_REF_RE = re.compile( r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$" ) @@ -129,7 +132,16 @@ def validate_deploy(value, location): deploy = require_mapping(value, location) reject_unknown( deploy, - {"flightdeck_ref", "env_ref", "extra_refs", "host", "credentials", "keep_releases"}, + { + "flightdeck_ref", + "env_ref", + "extra_refs", + "hosts", + "path", + "sops_age_key_file", + "credentials", + "keep_releases", + }, location, ) flightdeck_ref = require_string(deploy, "flightdeck_ref", location) @@ -146,27 +158,28 @@ def validate_deploy(value, location): keep_releases = deploy.get("keep_releases", 5) if not isinstance(keep_releases, int) or isinstance(keep_releases, bool) or keep_releases < 1: raise TargetError(f"{location}.keep_releases must be a positive integer") - host = require_mapping(deploy.get("host"), f"{location}.host") - reject_unknown(host, {"addresses", "user", "path", "sops_age_key_file"}, f"{location}.host") - addresses = host.get("addresses") - if not isinstance(addresses, list) or not addresses: - raise TargetError(f"{location}.host.addresses must be a non-empty array") - if any(not isinstance(item, str) or not item for item in addresses): - raise TargetError(f"{location}.host.addresses must contain non-empty strings") - user = require_string(host, "user", f"{location}.host", "root") - path = require_string(host, "path", f"{location}.host", "~/flightdeck") + hosts = deploy.get("hosts") + if not isinstance(hosts, list) or not hosts: + raise TargetError(f"{location}.hosts must be a non-empty array") + destinations = [] + for destination in hosts: + if not isinstance(destination, str) or not SSH_DESTINATION_RE.fullmatch(destination): + raise TargetError(f"{location}.hosts contains an invalid user@host destination") + destinations.append(SSH_DESTINATION_RE.fullmatch(destination).group("host")) + if len(destinations) != len(set(destinations)): + raise TargetError(f"{location}.hosts contains duplicate host addresses") + path = require_string(deploy, "path", location, "~/flightdeck") sops_key_file = require_string( - host, + deploy, "sops_age_key_file", - f"{location}.host", + location, "~/.config/sops/age/keys.txt", ) result = { "flightdeck_ref": flightdeck_ref, "env_ref": env_ref, "extra_refs": extra_refs, - "hosts": addresses, - "user": user, + "hosts": hosts, "path": path, "keep_releases": keep_releases, "sops_age_key_file": sops_key_file, diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py index 718b20e..8d0b1a8 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -21,11 +21,9 @@ env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env extra_refs: - rubykatzen/apps@v3.0.0:flightdeck-extra.zip - host: - addresses: [100.75.50.2] - user: rubykatzen-com - path: ~/flightdeck - sops_age_key_file: ~/.config/sops/age/keys.txt + hosts: [rubykatzen-com@100.75.50.2] + path: ~/flightdeck + sops_age_key_file: ~/.config/sops/age/keys.txt credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID @@ -59,7 +57,7 @@ def test_builds_encrypt_matrix(self): def test_builds_deploy_matrix(self): item = load_targets.build_matrix(self.directory, "deploy")["include"][0] - self.assertEqual(item["hosts"], ["100.75.50.2"]) + self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) self.assertEqual(item["path"], "~/flightdeck") @@ -70,8 +68,8 @@ def test_builds_deploy_matrix(self): def test_allows_host_default_overrides(self): path = self.directory / "hawkeye.yml" path.write_text( - TARGET.replace(" path: ~/flightdeck\n", "").replace( - " sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" + TARGET.replace(" path: ~/flightdeck\n", "").replace( + " sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" ) ) item = load_targets.build_matrix(self.directory, "deploy")["include"][0] @@ -79,10 +77,10 @@ def test_allows_host_default_overrides(self): self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") path.write_text( - TARGET.replace(" path: ~/flightdeck", " path: ~/custom") + TARGET.replace(" path: ~/flightdeck", " path: ~/custom") .replace( - " sops_age_key_file: ~/.config/sops/age/keys.txt", - " sops_age_key_file: ~/.config/sops/age/custom.txt", + " sops_age_key_file: ~/.config/sops/age/keys.txt", + " sops_age_key_file: ~/.config/sops/age/custom.txt", ) ) item = load_targets.build_matrix(self.directory, "deploy")["include"][0] @@ -103,10 +101,21 @@ def test_rejects_missing_selected_section(self): with self.assertRaisesRegex(load_targets.TargetError, "has no deploy section"): load_targets.build_matrix(self.directory, "deploy", "encrypt-only") - def test_rejects_string_addresses(self): + def test_rejects_invalid_ssh_destination(self): path = self.directory / "hawkeye.yml" - path.write_text(TARGET.replace("addresses: [100.75.50.2]", 'addresses: "100.75.50.2"')) - with self.assertRaisesRegex(load_targets.TargetError, "addresses must be a non-empty array"): + path.write_text(TARGET.replace("rubykatzen-com@100.75.50.2", "100.75.50.2")) + with self.assertRaisesRegex(load_targets.TargetError, "invalid user@host destination"): + load_targets.build_matrix(self.directory, "deploy") + + def test_rejects_duplicate_host_addresses(self): + path = self.directory / "hawkeye.yml" + path.write_text( + TARGET.replace( + "hosts: [rubykatzen-com@100.75.50.2]", + "hosts: [first@100.75.50.2, second@100.75.50.2]", + ) + ) + with self.assertRaisesRegex(load_targets.TargetError, "duplicate host addresses"): load_targets.build_matrix(self.directory, "deploy") def test_rejects_apps_in_env(self): diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 58a457c..7bdf79a 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -3,13 +3,9 @@ on: workflow_call: inputs: hosts: - description: JSON array of host addresses to deploy to. + description: JSON array of user@host SSH destinations to deploy to. type: string required: true - user: - description: SSH user for the Ansible connection. - type: string - default: root app-ref: description: Full release ref of the Flightdeck bundle to deploy, in owner/repo@tag format. type: string @@ -31,9 +27,9 @@ on: type: number default: 5 sops-age-key-file: - description: Path to the server-local SOPS age key file. Defaults to ~/.config/sops/age/keys.txt for the connecting user. + description: Path to the server-local SOPS age key file, relative to each SSH user's home when it starts with ~. type: string - default: "" + default: "~/.config/sops/age/keys.txt" tailscale-oauth-client-id: description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). type: string @@ -84,33 +80,25 @@ jobs: DEPLOY_PATH: ${{ inputs.path }} KEEP_RELEASES: ${{ inputs.keep-releases }} SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} - SSH_USER: ${{ inputs.user }} run: | - if [ "$SSH_USER" = root ]; then - home_dir=/root - else - home_dir="/home/$SSH_USER" - fi - deploy_path="${DEPLOY_PATH/#\~/$home_dir}" - sops_key_file="$SOPS_KEY_FILE" - if [ -z "$sops_key_file" ]; then - sops_key_file="$home_dir/.config/sops/age/keys.txt" - else - sops_key_file="${sops_key_file/#\~/$home_dir}" - fi - hosts_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("hosts must be a non-empty string array") end' <<< "$HOSTS")" + hosts_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and test("^[a-z_][a-z0-9_-]*@[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$")) then . else error("hosts must be a non-empty user@host string array") end' <<< "$HOSTS")" extra_refs_json="$(jq -ce 'if type == "array" and all(.[]; type == "string" and length > 0) then . else error("extra-refs must be a string array") end' <<< "$EXTRA_REFS")" - inventory="$(jq -r 'join(",") + ","' <<< "$hosts_json")" + jq -ce ' + reduce .[] as $destination ({all: {hosts: {}}}; + ($destination | capture("^(?[^@]+)@(?.+)$")) as $ssh | + .all.hosts[$ssh.host] = {ansible_user: $ssh.user} + ) + ' <<< "$hosts_json" > "$RUNNER_TEMP/flightdeck-inventory.json" json="$(jq -n \ --arg app_ref "$APP_REF" \ --arg env_ref "$ENV_REF" \ --argjson extra_refs "$extra_refs_json" \ - --arg path "$deploy_path" \ + --arg path "$DEPLOY_PATH" \ --argjson keep_releases "$KEEP_RELEASES" \ - --arg sops_key_file "$sops_key_file" \ + --arg sops_key_file "$SOPS_KEY_FILE" \ '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_extra_refs: $extra_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" echo "json=$json" >> "$GITHUB_OUTPUT" - echo "inventory=$inventory" >> "$GITHUB_OUTPUT" + echo "inventory=$RUNNER_TEMP/flightdeck-inventory.json" >> "$GITHUB_OUTPUT" - name: Run playbook shell: bash env: @@ -118,5 +106,4 @@ jobs: run: | ansible-playbook ansible/deploy.yml \ -i "${{ steps.vars.outputs.inventory }}" \ - -u "${{ inputs.user }}" \ -e "${{ steps.vars.outputs.json }}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 95765aa..890ed8f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,6 @@ jobs: uses: $/.github/workflows/deploy-shared.yml with: hosts: ${{ toJson(matrix.hosts) }} - user: ${{ matrix.user }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} extra-refs: ${{ toJson(matrix.extra_refs) }} diff --git a/README.md b/README.md index 95cba1a..881c3e9 100644 --- a/README.md +++ b/README.md @@ -463,13 +463,11 @@ deploy: env_ref: owner/config@latest:mainframe.sops.env extra_refs: - owner/extra-apps@latest - host: - addresses: - - 100.64.0.1 - - 100.64.0.2 - user: deploy - path: ~/flightdeck - sops_age_key_file: ~/.config/sops/age/keys.txt + hosts: + - deploy@100.64.0.1 + - deploy@100.64.0.2 + path: ~/flightdeck + sops_age_key_file: ~/.config/sops/age/keys.txt credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID @@ -478,7 +476,7 @@ deploy: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `host.addresses` are YAML arrays. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. --- @@ -553,8 +551,7 @@ jobs: deploy: uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v1.2.3 with: - hosts: '["100.64.0.1", "100.64.0.2"]' # required JSON array - user: root # default: root + hosts: '["deploy@100.64.0.1", "deploy@100.64.0.2"]' # required JSON array app-ref: rubykatzen/flightdeck@latest # required full release ref env-ref: "${{ github.repository }}@latest:.sops.env" # required # extra-refs: '["owner/repo@latest"]' # optional JSON array, default: [] diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 83a7fe0..1a96bd8 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -16,6 +16,13 @@ - flightdeck_keep_releases is defined - flightdeck_sops_age_key_file is defined and flightdeck_sops_age_key_file | length > 0 fail_msg: "Set flightdeck_app_ref, flightdeck_env_ref, flightdeck_extra_refs, flightdeck_path, flightdeck_keep_releases, and flightdeck_sops_age_key_file" + - name: Resolve Flightdeck user paths + set_fact: + flightdeck_user_home: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}" + - name: Expand Flightdeck user paths + set_fact: + flightdeck_path: "{{ flightdeck_path | regex_replace('^~', flightdeck_user_home) }}" + flightdeck_sops_age_key_file: "{{ flightdeck_sops_age_key_file | regex_replace('^~', flightdeck_user_home) }}" - name: Apply Flightdeck paths set_fact: flightdeck_release_name: "{{ ansible_facts['date_time'].iso8601_basic_short }}" diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 4fb9219..dfbe5ee 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -19,10 +19,8 @@ deploy: flightdeck_ref: rubykatzen/flightdeck@latest env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env extra_refs: [] - host: - addresses: - - 100.75.50.2 - user: rubykatzen-com + hosts: + - rubykatzen-com@100.75.50.2 credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID From f1100c2f53f8341d902c2b876bbfe8e5cc136bbd Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 14:21:18 +0200 Subject: [PATCH 25/35] refactor: separate encryption configs from deploy targets --- .github/actions/encrypt-env/README.md | 25 ++-- .../actions/encrypt-env/scripts/render-env.py | 21 ++- .../encrypt-env/tests/test_render_env.py | 50 ++++---- .github/actions/load-targets/README.md | 8 +- .github/actions/load-targets/action.yml | 15 ++- .../load-targets/scripts/load-targets.py | 29 ++--- .../load-targets/tests/test_load_targets.py | 121 +++++++++--------- AGENTS.md | 2 +- README.md | 85 ++++++------ encrypt/hawkeye.yml | 16 +++ targets/hawkeye.yml | 40 ++---- 11 files changed, 203 insertions(+), 209 deletions(-) create mode 100644 encrypt/hawkeye.yml diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md index afa3698..ec9214e 100644 --- a/.github/actions/encrypt-env/README.md +++ b/.github/actions/encrypt-env/README.md @@ -1,6 +1,6 @@ # encrypt-env -Composite GitHub Action that renders the `encrypt` section of a target from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` to an existing GitHub Release. +Composite GitHub Action that renders an encryption config from GitHub Secrets/Variables, encrypts it for named age recipients, and uploads `.sops.env` to an existing GitHub Release. The release must exist before this action runs. @@ -9,7 +9,7 @@ The release must exist before this action runs. ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: targets/mainframe.yml # required + manifest: encrypt/mainframe.yml # required keys-directory: keys # default: keys release-tag: ${{ needs.release.outputs.tag }} # required, must already exist release-repo: "" # default: current repository @@ -29,16 +29,15 @@ permissions: ## Manifest ```yaml -encrypt: - asset: mainframe.sops.env - keys: - - mainframe - apps: - - traefik - - rybbit - env: - APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name - APPS_TIMEZONE: APPS_TIMEZONE +asset: mainframe.sops.env +keys: + - mainframe +apps: + - traefik + - rybbit +env: + APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name + APPS_TIMEZONE: APPS_TIMEZONE ``` -The action renders `encrypt.apps` as the comma-separated `APPS` dotenv value. For each name in `encrypt.keys`, it loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. +The action renders `apps` as the comma-separated `APPS` dotenv value. For each name in `keys`, it loads `/.pub`. Secrets take precedence over Variables when both contain the same source key. Every source key must resolve or the action fails. diff --git a/.github/actions/encrypt-env/scripts/render-env.py b/.github/actions/encrypt-env/scripts/render-env.py index f629c34..1bf667a 100644 --- a/.github/actions/encrypt-env/scripts/render-env.py +++ b/.github/actions/encrypt-env/scripts/render-env.py @@ -58,18 +58,15 @@ def load_manifest(path): raise ManifestError(f"{path} is not valid YAML: {exc}") from exc if not isinstance(manifest, dict): raise ManifestError(f"{path} must contain a YAML mapping") - encrypt = manifest.get("encrypt") - if not isinstance(encrypt, dict): - raise ManifestError(f"{path} must contain an encrypt mapping") - unknown = sorted(set(encrypt) - {"asset", "keys", "apps", "env"}) + unknown = sorted(set(manifest) - {"asset", "keys", "apps", "env"}) if unknown: - raise ManifestError("encrypt contains unknown keys: " + ", ".join(unknown)) - asset = encrypt.get("asset") - keys = encrypt.get("keys") - apps = encrypt.get("apps") - env = encrypt.get("env") + raise ManifestError("manifest contains unknown keys: " + ", ".join(unknown)) + asset = manifest.get("asset") + keys = manifest.get("keys") + apps = manifest.get("apps") + env = manifest.get("env") if not isinstance(asset, str) or not ASSET_RE.fullmatch(asset): - raise ManifestError("encrypt.asset must be named like server.sops.env") + raise ManifestError("asset must be named like server.sops.env") if not isinstance(keys, list) or not keys: raise ManifestError("keys must be a non-empty list") if not isinstance(apps, list) or not apps: @@ -85,13 +82,13 @@ def load_manifest(path): if len(apps) != len(set(apps)): raise ManifestError("apps contains duplicate app names") if "APPS" in env: - raise ManifestError("APPS must be configured through encrypt.apps") + raise ManifestError("APPS must be configured through apps") for output_name, source_name in env.items(): if not isinstance(output_name, str) or not ENV_NAME_RE.fullmatch(output_name): raise ManifestError(f"invalid output env name: {output_name!r}") if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): raise ManifestError(f"invalid source key for {output_name}: {source_name!r}") - return encrypt + return manifest def dotenv_value(value): diff --git a/.github/actions/encrypt-env/tests/test_render_env.py b/.github/actions/encrypt-env/tests/test_render_env.py index 062fad0..6fe5d6f 100644 --- a/.github/actions/encrypt-env/tests/test_render_env.py +++ b/.github/actions/encrypt-env/tests/test_render_env.py @@ -31,30 +31,26 @@ def test_quotes_shell_sensitive_values(self): def test_rejects_raw_env(self): with self.assertRaises(render_env.ManifestError): - render_env.load_manifest( - self.write_manifest("encrypt:\n asset: test.sops.env\n raw_env: [APPS]\n") - ) + render_env.load_manifest(self.write_manifest("asset: test.sops.env\nraw_env: [APPS]\n")) def test_rejects_apps_in_env(self): manifest = ( - "encrypt:\n" - " asset: test.sops.env\n" - " keys: [test]\n" - " apps: [traefik]\n" - " env:\n" - " APPS: TEST_APPS\n" + "asset: test.sops.env\n" + "keys: [test]\n" + "apps: [traefik]\n" + "env:\n" + " APPS: TEST_APPS\n" ) with self.assertRaisesRegex(render_env.ManifestError, "must be configured through"): render_env.load_manifest(self.write_manifest(manifest)) def test_rejects_duplicate_apps(self): manifest = ( - "encrypt:\n" - " asset: test.sops.env\n" - " keys: [test]\n" - " apps: [traefik, traefik]\n" - " env:\n" - " TOKEN: TOKEN\n" + "asset: test.sops.env\n" + "keys: [test]\n" + "apps: [traefik, traefik]\n" + "env:\n" + " TOKEN: TOKEN\n" ) with self.assertRaisesRegex(render_env.ManifestError, "duplicate app names"): render_env.load_manifest(self.write_manifest(manifest)) @@ -67,13 +63,12 @@ def test_duplicate_yaml_keys_fail(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "manifest.yml" path.write_text( - "encrypt:\n" - " asset: mainframe.sops.env\n" - " keys: [master, server]\n" - " apps: [traefik, rybbit]\n" - " env:\n" - " TOKEN: FIRST\n" - " TOKEN: SECOND\n" + "asset: mainframe.sops.env\n" + "keys: [master, server]\n" + "apps: [traefik, rybbit]\n" + "env:\n" + " TOKEN: FIRST\n" + " TOKEN: SECOND\n" ) with self.assertRaises(render_env.ManifestError): render_env.load_manifest(path) @@ -92,12 +87,11 @@ def test_main_writes_env_and_outputs(self): env_path = root / ".env" outputs_path = root / "outputs" manifest_path.write_text( - "encrypt:\n" - " asset: mainframe.sops.env\n" - " keys: [master, server]\n" - " apps: [traefik, rybbit]\n" - " env:\n" - " TOKEN: TOKEN\n" + "asset: mainframe.sops.env\n" + "keys: [master, server]\n" + "apps: [traefik, rybbit]\n" + "env:\n" + " TOKEN: TOKEN\n" ) old_env = os.environ.copy() os.environ.update( diff --git a/.github/actions/load-targets/README.md b/.github/actions/load-targets/README.md index 722f295..f9ff834 100644 --- a/.github/actions/load-targets/README.md +++ b/.github/actions/load-targets/README.md @@ -1,6 +1,6 @@ # load-targets -Composite GitHub Action that validates YAML files in `targets/` and builds a GitHub Actions matrix from either their `encrypt` or `deploy` sections. +Composite GitHub Action that validates encryption configs or deployment targets and builds a GitHub Actions matrix. ## Usage @@ -10,9 +10,9 @@ Composite GitHub Action that validates YAML files in `targets/` and builds a Git with: mode: deploy # encrypt or deploy target: all # default: all - directory: targets # default: targets + # directory: custom # optional; defaults to encrypt/ or targets/ based on mode ``` -The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. When `target` is `all`, files without the requested section are skipped. A specifically selected target must contain that section. +The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Encrypt mode validates the flat encryption schema; deploy mode validates the flat target schema. -Every target is fully validated before its requested section is added to the matrix. Target files may use either the `.yml` or `.yaml` extension, and their basename becomes the target name. +Every file is fully validated before it is added to the matrix. Files may use either the `.yml` or `.yaml` extension, and their basename becomes the matrix target name. diff --git a/.github/actions/load-targets/action.yml b/.github/actions/load-targets/action.yml index de87e15..6b4c98f 100644 --- a/.github/actions/load-targets/action.yml +++ b/.github/actions/load-targets/action.yml @@ -2,19 +2,19 @@ name: Load targets description: Validate target manifests and build an encrypt or deploy strategy matrix. inputs: mode: - description: Target section to load (encrypt or deploy). + description: Configuration type to load (encrypt or deploy). required: true target: description: Target name to load, or all. required: false default: all directory: - description: Directory containing target YAML files. + description: Directory containing YAML files. Defaults to encrypt for encrypt mode and targets for deploy mode. required: false - default: targets + default: "" outputs: matrix: - description: JSON strategy matrix containing validated target data. + description: JSON strategy matrix containing validated configuration data. value: ${{ steps.load.outputs.matrix }} count: description: Number of targets in the matrix. @@ -32,4 +32,9 @@ runs: DIRECTORY: ${{ inputs.directory }} MODE: ${{ inputs.mode }} TARGET: ${{ inputs.target }} - run: python3 "$GITHUB_ACTION_PATH/scripts/load-targets.py" --directory "$DIRECTORY" --mode "$MODE" --target "$TARGET" + run: | + directory_args=() + if [ -n "$DIRECTORY" ]; then + directory_args=(--directory "$DIRECTORY") + fi + python3 "$GITHUB_ACTION_PATH/scripts/load-targets.py" "${directory_args[@]}" --mode "$MODE" --target "$TARGET" diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-targets/scripts/load-targets.py index 16059c0..34b28a7 100644 --- a/.github/actions/load-targets/scripts/load-targets.py +++ b/.github/actions/load-targets/scripts/load-targets.py @@ -188,21 +188,15 @@ def validate_deploy(value, location): return result -def load_target(path): +def load_config(path, mode): try: value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) except yaml.YAMLError as exc: raise TargetError(f"{path} is not valid YAML: {exc}") from exc - target = require_mapping(value, str(path)) - reject_unknown(target, {"encrypt", "deploy"}, str(path)) - if not target: - raise TargetError(f"{path} must contain encrypt or deploy") - result = {} - if "encrypt" in target: - result["encrypt"] = validate_encrypt(target["encrypt"], f"{path}.encrypt") - if "deploy" in target: - result["deploy"] = validate_deploy(target["deploy"], f"{path}.deploy") - return result + config = require_mapping(value, str(path)) + if mode == "encrypt": + return validate_encrypt(config, str(path)) + return validate_deploy(config, str(path)) def build_matrix(directory, mode, selected="all"): @@ -223,15 +217,11 @@ def build_matrix(directory, mode, selected="all"): if name in seen_names: raise TargetError(f"duplicate target name: {name}") seen_names.add(name) - target = load_target(path) + config = load_config(path, mode) if selected != "all" and name != selected: continue - if mode not in target: - if selected != "all": - raise TargetError(f"target {name} has no {mode} section") - continue item = {"target": name, "manifest": str(path)} - item.update(target[mode]) + item.update(config) include.append(item) return {"include": include} @@ -245,12 +235,13 @@ def write_github_output(name, value): def main(argv=None): parser = argparse.ArgumentParser() - parser.add_argument("--directory", type=Path, default=Path("targets")) + parser.add_argument("--directory", type=Path) parser.add_argument("--mode", required=True) parser.add_argument("--target", default="all") args = parser.parse_args(argv) try: - matrix = build_matrix(args.directory, args.mode, args.target) + directory = args.directory or Path("encrypt" if args.mode == "encrypt" else "targets") + matrix = build_matrix(directory, args.mode, args.target) encoded = json.dumps(matrix, separators=(",", ":")) write_github_output("matrix", encoded) write_github_output("count", len(matrix["include"])) diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-targets/tests/test_load_targets.py index 8d0b1a8..1d7eee3 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-targets/tests/test_load_targets.py @@ -9,27 +9,28 @@ SPEC.loader.exec_module(load_targets) +ENCRYPT = """\ +asset: hawkeye.sops.env +keys: [hawkeye] +apps: [traefik, rybbit] +env: + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN +""" + TARGET = """\ -encrypt: - asset: hawkeye.sops.env - keys: [hawkeye] - apps: [traefik, rybbit] - env: - APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN -deploy: - flightdeck_ref: rubykatzen/flightdeck@v1.2.3 - env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env - extra_refs: - - rubykatzen/apps@v3.0.0:flightdeck-extra.zip - hosts: [rubykatzen-com@100.75.50.2] - path: ~/flightdeck - sops_age_key_file: ~/.config/sops/age/keys.txt - credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET +flightdeck_ref: rubykatzen/flightdeck@v1.2.3 +env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env +extra_refs: + - rubykatzen/apps@v3.0.0:flightdeck-extra.zip +hosts: [rubykatzen-com@100.75.50.2] +path: ~/flightdeck +sops_age_key_file: ~/.config/sops/age/keys.txt +credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET """ @@ -37,18 +38,23 @@ class LoadTargetsTest(unittest.TestCase): def setUp(self): self.temporary_directory = tempfile.TemporaryDirectory() self.addCleanup(self.temporary_directory.cleanup) - self.directory = Path(self.temporary_directory.name) - (self.directory / "hawkeye.yml").write_text(TARGET) + self.root = Path(self.temporary_directory.name) + self.encrypt_directory = self.root / "encrypt" + self.targets_directory = self.root / "targets" + self.encrypt_directory.mkdir() + self.targets_directory.mkdir() + (self.encrypt_directory / "hawkeye.yml").write_text(ENCRYPT) + (self.targets_directory / "hawkeye.yml").write_text(TARGET) def test_builds_encrypt_matrix(self): - matrix = load_targets.build_matrix(self.directory, "encrypt") + matrix = load_targets.build_matrix(self.encrypt_directory, "encrypt") self.assertEqual( matrix, { "include": [ { "target": "hawkeye", - "manifest": str(self.directory / "hawkeye.yml"), + "manifest": str(self.encrypt_directory / "hawkeye.yml"), "asset": "hawkeye.sops.env", } ] @@ -56,59 +62,58 @@ def test_builds_encrypt_matrix(self): ) def test_builds_deploy_matrix(self): - item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) self.assertEqual(item["path"], "~/flightdeck") self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") - self.assertNotIn("apps", item) - def test_allows_host_default_overrides(self): - path = self.directory / "hawkeye.yml" + def test_allows_deploy_path_default_overrides(self): + path = self.targets_directory / "hawkeye.yml" path.write_text( - TARGET.replace(" path: ~/flightdeck\n", "").replace( - " sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" + TARGET.replace("path: ~/flightdeck\n", "").replace( + "sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" ) ) - item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["path"], "~/flightdeck") self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") path.write_text( - TARGET.replace(" path: ~/flightdeck", " path: ~/custom") - .replace( - " sops_age_key_file: ~/.config/sops/age/keys.txt", - " sops_age_key_file: ~/.config/sops/age/custom.txt", + TARGET.replace("path: ~/flightdeck", "path: ~/custom").replace( + "sops_age_key_file: ~/.config/sops/age/keys.txt", + "sops_age_key_file: ~/.config/sops/age/custom.txt", ) ) - item = load_targets.build_matrix(self.directory, "deploy")["include"][0] + item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["path"], "~/custom") self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/custom.txt") - def test_filters_selected_target(self): - (self.directory / "other.yml").write_text(TARGET.replace("hawkeye.sops.env", "other.sops.env")) - matrix = load_targets.build_matrix(self.directory, "encrypt", "hawkeye") + def test_filters_selected_config(self): + (self.encrypt_directory / "other.yml").write_text( + ENCRYPT.replace("hawkeye.sops.env", "other.sops.env") + ) + matrix = load_targets.build_matrix(self.encrypt_directory, "encrypt", "hawkeye") self.assertEqual([item["target"] for item in matrix["include"]], ["hawkeye"]) - def test_rejects_unknown_target(self): + def test_rejects_unknown_config(self): with self.assertRaisesRegex(load_targets.TargetError, "unknown target"): - load_targets.build_matrix(self.directory, "deploy", "missing") + load_targets.build_matrix(self.targets_directory, "deploy", "missing") - def test_rejects_missing_selected_section(self): - (self.directory / "encrypt-only.yml").write_text(TARGET.split("deploy:\n", maxsplit=1)[0]) - with self.assertRaisesRegex(load_targets.TargetError, "has no deploy section"): - load_targets.build_matrix(self.directory, "deploy", "encrypt-only") + def test_rejects_wrong_config_type(self): + with self.assertRaisesRegex(load_targets.TargetError, "unknown keys"): + load_targets.build_matrix(self.encrypt_directory, "deploy") def test_rejects_invalid_ssh_destination(self): - path = self.directory / "hawkeye.yml" + path = self.targets_directory / "hawkeye.yml" path.write_text(TARGET.replace("rubykatzen-com@100.75.50.2", "100.75.50.2")) with self.assertRaisesRegex(load_targets.TargetError, "invalid user@host destination"): - load_targets.build_matrix(self.directory, "deploy") + load_targets.build_matrix(self.targets_directory, "deploy") def test_rejects_duplicate_host_addresses(self): - path = self.directory / "hawkeye.yml" + path = self.targets_directory / "hawkeye.yml" path.write_text( TARGET.replace( "hosts: [rubykatzen-com@100.75.50.2]", @@ -116,25 +121,25 @@ def test_rejects_duplicate_host_addresses(self): ) ) with self.assertRaisesRegex(load_targets.TargetError, "duplicate host addresses"): - load_targets.build_matrix(self.directory, "deploy") + load_targets.build_matrix(self.targets_directory, "deploy") def test_rejects_apps_in_env(self): - path = self.directory / "hawkeye.yml" - path.write_text(TARGET.replace(" APPS_DOMAIN:", " APPS: RUBYKATZEN_COM_APPS\n APPS_DOMAIN:")) + path = self.encrypt_directory / "hawkeye.yml" + path.write_text(ENCRYPT.replace(" APPS_DOMAIN:", " APPS: RUBYKATZEN_COM_APPS\n APPS_DOMAIN:")) with self.assertRaisesRegex(load_targets.TargetError, "must be configured through"): - load_targets.build_matrix(self.directory, "encrypt") + load_targets.build_matrix(self.encrypt_directory, "encrypt") def test_rejects_duplicate_apps(self): - path = self.directory / "hawkeye.yml" - path.write_text(TARGET.replace("apps: [traefik, rybbit]", "apps: [traefik, rybbit, traefik]")) + path = self.encrypt_directory / "hawkeye.yml" + path.write_text(ENCRYPT.replace("apps: [traefik, rybbit]", "apps: [traefik, rybbit, traefik]")) with self.assertRaisesRegex(load_targets.TargetError, "duplicate app names"): - load_targets.build_matrix(self.directory, "encrypt") + load_targets.build_matrix(self.encrypt_directory, "encrypt") def test_rejects_duplicate_keys(self): - path = self.directory / "hawkeye.yml" - path.write_text(TARGET.replace(" asset: hawkeye.sops.env", " asset: one.sops.env\n asset: two.sops.env")) + path = self.encrypt_directory / "hawkeye.yml" + path.write_text(ENCRYPT.replace("asset: hawkeye.sops.env", "asset: one.sops.env\nasset: two.sops.env")) with self.assertRaisesRegex(load_targets.TargetError, "duplicate YAML key"): - load_targets.build_matrix(self.directory, "encrypt") + load_targets.build_matrix(self.encrypt_directory, "encrypt") if __name__ == "__main__": diff --git a/AGENTS.md b/AGENTS.md index fd09224..4f5584b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,7 +409,7 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release Deployment helpers live in this repository: - `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` -- `.github/actions/encrypt-env/` is a local composite action for rendering target env manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset +- `.github/actions/encrypt-env/` is a local composite action for rendering `encrypt/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. diff --git a/README.md b/README.md index 881c3e9..bfeb250 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,8 @@ flightdeck/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ -├── targets/ # Independent encryption and deployment targets +├── encrypt/ # Encrypted env asset configurations +├── targets/ # Deployment targets ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template │ @@ -444,50 +445,55 @@ This repository provides three composite actions under `.github/actions/` (`buil --- -### Targets +### Encrypt Configurations And Targets -Each file in `targets/` describes one logical target. Its optional root sections are independent: an encryption-only repository can omit `deploy`, while a deployment-only repository can omit `encrypt`. A target containing both ties the encrypted env asset and deployment hosts together by name. +Files in `encrypt/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. + +`encrypt/mainframe.yml`: ```yaml -encrypt: - asset: mainframe.sops.env - keys: - - mainframe - apps: - - traefik - - rybbit - env: - APPS_DOMAIN: MAINFRAME_DOMAIN -deploy: - flightdeck_ref: rubykatzen/flightdeck@latest - env_ref: owner/config@latest:mainframe.sops.env - extra_refs: - - owner/extra-apps@latest - hosts: - - deploy@100.64.0.1 - - deploy@100.64.0.2 - path: ~/flightdeck - sops_age_key_file: ~/.config/sops/age/keys.txt - credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET +asset: mainframe.sops.env +keys: + - mainframe +apps: + - traefik + - rybbit +env: + APPS_DOMAIN: MAINFRAME_DOMAIN +``` + +`targets/mainframe.yml`: + +```yaml +flightdeck_ref: rubykatzen/flightdeck@latest +env_ref: owner/config@latest:mainframe.sops.env +extra_refs: + - owner/extra-apps@latest +hosts: + - deploy@100.64.0.1 + - deploy@100.64.0.2 +path: ~/flightdeck +sops_age_key_file: ~/.config/sops/age/keys.txt +credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `encrypt.apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates every target and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates each collection and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. --- ### `encrypt-env` -Renders the `encrypt` section of a target from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` to an existing GitHub Release. Release creation remains the calling workflow's responsibility. +Renders an encryption config from GitHub Secrets/Variables, encrypts it with SOPS age recipients, and uploads `.sops.env` to an existing GitHub Release. Release creation remains the calling workflow's responsibility. ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: targets/mainframe.yml # required + manifest: encrypt/mainframe.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository @@ -502,15 +508,14 @@ Requires `contents: write` permission on the calling job. **Manifest format:** ```yaml -encrypt: - asset: mainframe.sops.env - keys: - - mainframe - apps: - - traefik - - rybbit - env: - APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name +asset: mainframe.sops.env +keys: + - mainframe +apps: + - traefik + - rybbit +env: + APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name ``` Secrets take precedence over Variables when both contain the same source key. Every source key must exist or the action fails. diff --git a/encrypt/hawkeye.yml b/encrypt/hawkeye.yml new file mode 100644 index 0000000..fb84284 --- /dev/null +++ b/encrypt/hawkeye.yml @@ -0,0 +1,16 @@ +asset: hawkeye.sops.env +keys: + - hawkeye +apps: + - traefik + - rybbit +env: + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN + APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL + APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER + APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN + APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD + APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 + APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE + TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT + TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index dfbe5ee..5f91afd 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,29 +1,11 @@ -encrypt: - asset: hawkeye.sops.env - keys: - - hawkeye - apps: - - traefik - - rybbit - env: - APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN - APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL - APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER - APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN - APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD - APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 - APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE - TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT - TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT -deploy: - flightdeck_ref: rubykatzen/flightdeck@latest - env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env - extra_refs: [] - hosts: - - rubykatzen-com@100.75.50.2 - credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET +flightdeck_ref: rubykatzen/flightdeck@latest +env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env +extra_refs: [] +hosts: + - rubykatzen-com@100.75.50.2 +credentials: + variables: + tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID + secrets: + ssh_private_key: DEPLOY_SSH_PRIVATE_KEY + tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET From 2c1f604d240b15d6ed525244f0c3d6a0727d1d7f Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 22:35:31 +0200 Subject: [PATCH 26/35] fix: gate automatic deploy on an actual release, not any Release run deploy.yml previously fired on every completed "Release" workflow_run, including runs where release-please created no release - which meant every ordinary merge to main would redeploy hawkeye and bounce traefik/rybbit. Move the automatic deploy into release.yml, gated on release_created like upload/targets/encrypt, and leave deploy.yml with only the manual workflow_dispatch redeploy path. --- .github/workflows/deploy.yml | 8 -------- .github/workflows/release.yml | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 890ed8f..4ed261d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,22 +6,14 @@ on: description: Deploy target to redeploy. Use "all" to redeploy every target. type: string default: all - workflow_run: - workflows: [Release] - types: [completed] jobs: targets: - if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') runs-on: ubuntu-latest outputs: matrix: ${{ steps.targets.outputs.matrix }} count: ${{ steps.targets.outputs.count }} steps: - uses: actions/checkout@v7 - with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} - uses: $/.github/actions/load-targets id: targets with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a42bba8..801a0ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,3 +76,36 @@ jobs: env: GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} GITHUB_VARS_JSON: ${{ toJson(vars) }} + deploy-targets: + needs: release + if: needs.release.outputs.release_created == 'true' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + count: ${{ steps.targets.outputs.count }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.release.outputs.tag_name }} + - uses: $/.github/actions/load-targets + id: targets + with: + mode: deploy + deploy: + needs: [upload, encrypt, deploy-targets] + if: needs.deploy-targets.outputs.count != '0' + strategy: + matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} + uses: $/.github/workflows/deploy-shared.yml + with: + hosts: ${{ toJson(matrix.hosts) }} + app-ref: ${{ matrix.flightdeck_ref }} + env-ref: ${{ matrix.env_ref }} + extra-refs: ${{ toJson(matrix.extra_refs) }} + path: ${{ matrix.path }} + keep-releases: ${{ matrix.keep_releases }} + sops-age-key-file: ${{ matrix.sops_age_key_file }} + tailscale-oauth-client-id: ${{ vars[matrix.tailscale_oauth_client_id_variable] }} + secrets: + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + tailscale-oauth-secret: ${{ secrets[matrix.tailscale_oauth_secret] }} From c6d01db1eb2ba2bcb306995828c7acd86c035bfd Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 22:45:57 +0200 Subject: [PATCH 27/35] refactor: rename load-targets to load-matrix "targets" already means deploy destinations (servers). Reusing that name for the generic config-matrix reader used by both encrypt and deploy modes was confusing. Rename the action to load-matrix, its name-selector input/flag from target to name, and the matrix item's target field to name. Also rename release.yml's encrypt-mode job from targets to encrypt-configs so it doesn't collide semantically with the new deploy-targets job. --- .../{load-targets => load-matrix}/README.md | 10 +-- .../{load-targets => load-matrix}/action.yml | 14 ++-- .../requirements.txt | 0 .../scripts/load-matrix.py} | 74 +++++++++---------- .../tests/test_load_matrix.py} | 52 ++++++------- .github/workflows/deploy.yml | 4 +- .github/workflows/release.yml | 24 +++--- README.md | 6 +- 8 files changed, 92 insertions(+), 92 deletions(-) rename .github/actions/{load-targets => load-matrix}/README.md (76%) rename .github/actions/{load-targets => load-matrix}/action.yml (76%) rename .github/actions/{load-targets => load-matrix}/requirements.txt (100%) rename .github/actions/{load-targets/scripts/load-targets.py => load-matrix/scripts/load-matrix.py} (73%) rename .github/actions/{load-targets/tests/test_load_targets.py => load-matrix/tests/test_load_matrix.py} (69%) diff --git a/.github/actions/load-targets/README.md b/.github/actions/load-matrix/README.md similarity index 76% rename from .github/actions/load-targets/README.md rename to .github/actions/load-matrix/README.md index f9ff834..5419a85 100644 --- a/.github/actions/load-targets/README.md +++ b/.github/actions/load-matrix/README.md @@ -1,18 +1,18 @@ -# load-targets +# load-matrix Composite GitHub Action that validates encryption configs or deployment targets and builds a GitHub Actions matrix. ## Usage ```yaml -- uses: rubykatzen/flightdeck/.github/actions/load-targets@main - id: targets +- uses: rubykatzen/flightdeck/.github/actions/load-matrix@main + id: matrix with: mode: deploy # encrypt or deploy - target: all # default: all + name: all # default: all # directory: custom # optional; defaults to encrypt/ or targets/ based on mode ``` The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Encrypt mode validates the flat encryption schema; deploy mode validates the flat target schema. -Every file is fully validated before it is added to the matrix. Files may use either the `.yml` or `.yaml` extension, and their basename becomes the matrix target name. +Every file is fully validated before it is added to the matrix. Files may use either the `.yml` or `.yaml` extension, and their basename becomes the matrix item's `name`. diff --git a/.github/actions/load-targets/action.yml b/.github/actions/load-matrix/action.yml similarity index 76% rename from .github/actions/load-targets/action.yml rename to .github/actions/load-matrix/action.yml index 6b4c98f..86ae25d 100644 --- a/.github/actions/load-targets/action.yml +++ b/.github/actions/load-matrix/action.yml @@ -1,11 +1,11 @@ -name: Load targets -description: Validate target manifests and build an encrypt or deploy strategy matrix. +name: Load matrix +description: Validate config manifests and build an encrypt or deploy strategy matrix. inputs: mode: description: Configuration type to load (encrypt or deploy). required: true - target: - description: Target name to load, or all. + name: + description: Single manifest name to load, or all. required: false default: all directory: @@ -25,16 +25,16 @@ runs: - name: Install dependencies shell: bash run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" - - name: Load targets + - name: Load matrix id: load shell: bash env: DIRECTORY: ${{ inputs.directory }} MODE: ${{ inputs.mode }} - TARGET: ${{ inputs.target }} + NAME: ${{ inputs.name }} run: | directory_args=() if [ -n "$DIRECTORY" ]; then directory_args=(--directory "$DIRECTORY") fi - python3 "$GITHUB_ACTION_PATH/scripts/load-targets.py" "${directory_args[@]}" --mode "$MODE" --target "$TARGET" + python3 "$GITHUB_ACTION_PATH/scripts/load-matrix.py" "${directory_args[@]}" --mode "$MODE" --name "$NAME" diff --git a/.github/actions/load-targets/requirements.txt b/.github/actions/load-matrix/requirements.txt similarity index 100% rename from .github/actions/load-targets/requirements.txt rename to .github/actions/load-matrix/requirements.txt diff --git a/.github/actions/load-targets/scripts/load-targets.py b/.github/actions/load-matrix/scripts/load-matrix.py similarity index 73% rename from .github/actions/load-targets/scripts/load-targets.py rename to .github/actions/load-matrix/scripts/load-matrix.py index 34b28a7..d6128b0 100644 --- a/.github/actions/load-targets/scripts/load-targets.py +++ b/.github/actions/load-matrix/scripts/load-matrix.py @@ -8,7 +8,7 @@ import yaml -TARGET_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +MANIFEST_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") @@ -25,7 +25,7 @@ ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") -class TargetError(Exception): +class ManifestError(Exception): pass @@ -38,7 +38,7 @@ def construct_mapping(loader, node, deep=False): for key_node, value_node in node.value: key = loader.construct_object(key_node, deep=deep) if key in mapping: - raise TargetError(f"duplicate YAML key: {key}") + raise ManifestError(f"duplicate YAML key: {key}") mapping[key] = loader.construct_object(value_node, deep=deep) return mapping @@ -51,20 +51,20 @@ def construct_mapping(loader, node, deep=False): def require_mapping(value, location): if not isinstance(value, dict): - raise TargetError(f"{location} must be a mapping") + raise ManifestError(f"{location} must be a mapping") return value def reject_unknown(mapping, allowed, location): unknown = sorted(set(mapping) - set(allowed)) if unknown: - raise TargetError(f"{location} contains unknown keys: {', '.join(unknown)}") + raise ManifestError(f"{location} contains unknown keys: {', '.join(unknown)}") def require_string(mapping, key, location, default=None): value = mapping.get(key, default) if not isinstance(value, str) or not value: - raise TargetError(f"{location}.{key} must be a non-empty string") + raise ManifestError(f"{location}.{key} must be a non-empty string") return value @@ -73,29 +73,29 @@ def validate_encrypt(value, location): reject_unknown(encrypt, {"asset", "keys", "apps", "env"}, location) asset = require_string(encrypt, "asset", location) if not ASSET_RE.fullmatch(asset): - raise TargetError(f"{location}.asset must be named like server.sops.env") + raise ManifestError(f"{location}.asset must be named like server.sops.env") keys = encrypt.get("keys") if not isinstance(keys, list) or not keys: - raise TargetError(f"{location}.keys must be a non-empty array") + raise ManifestError(f"{location}.keys must be a non-empty array") if any(not isinstance(key, str) or not KEY_NAME_RE.fullmatch(key) for key in keys): - raise TargetError(f"{location}.keys contains an invalid age key name") + raise ManifestError(f"{location}.keys contains an invalid age key name") apps = encrypt.get("apps") if not isinstance(apps, list) or not apps: - raise TargetError(f"{location}.apps must be a non-empty array") + raise ManifestError(f"{location}.apps must be a non-empty array") if any(not isinstance(app, str) or not APP_NAME_RE.fullmatch(app) for app in apps): - raise TargetError(f"{location}.apps contains an invalid app name") + raise ManifestError(f"{location}.apps contains an invalid app name") if len(apps) != len(set(apps)): - raise TargetError(f"{location}.apps contains duplicate app names") + raise ManifestError(f"{location}.apps contains duplicate app names") env = require_mapping(encrypt.get("env"), f"{location}.env") if not env: - raise TargetError(f"{location}.env must not be empty") + raise ManifestError(f"{location}.env must not be empty") for output_name, source_name in env.items(): if not isinstance(output_name, str) or not ENV_NAME_RE.fullmatch(output_name): - raise TargetError(f"{location}.env contains an invalid output name: {output_name!r}") + raise ManifestError(f"{location}.env contains an invalid output name: {output_name!r}") if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): - raise TargetError(f"{location}.env.{output_name} has an invalid source name") + raise ManifestError(f"{location}.env.{output_name} has an invalid source name") if "APPS" in env: - raise TargetError(f"{location}.env.APPS must be configured through {location}.apps") + raise ManifestError(f"{location}.env.APPS must be configured through {location}.apps") return {"asset": asset} @@ -118,9 +118,9 @@ def validate_credentials(value, location): ("tailscale_oauth_secret", client_secret), ): if not isinstance(value_name, str): - raise TargetError(f"{location}.{name} must be a string") + raise ManifestError(f"{location}.{name} must be a string") if bool(client_id) != bool(client_secret): - raise TargetError(f"{location} must configure both Tailscale OAuth names or neither") + raise ManifestError(f"{location} must configure both Tailscale OAuth names or neither") return { "ssh_private_key_secret": ssh_private_key, "tailscale_oauth_client_id_variable": client_id, @@ -147,27 +147,27 @@ def validate_deploy(value, location): flightdeck_ref = require_string(deploy, "flightdeck_ref", location) env_ref = require_string(deploy, "env_ref", location) if not RELEASE_REF_RE.fullmatch(flightdeck_ref): - raise TargetError(f"{location}.flightdeck_ref must be in owner/repo@tag format") + raise ManifestError(f"{location}.flightdeck_ref must be in owner/repo@tag format") if not ENV_REF_RE.fullmatch(env_ref): - raise TargetError(f"{location}.env_ref must be in owner/repo@tag:asset.sops.env format") + raise ManifestError(f"{location}.env_ref must be in owner/repo@tag:asset.sops.env format") extra_refs = deploy.get("extra_refs", []) if not isinstance(extra_refs, list): - raise TargetError(f"{location}.extra_refs must be an array") + raise ManifestError(f"{location}.extra_refs must be an array") if any(not isinstance(ref, str) or not RELEASE_REF_RE.fullmatch(ref) for ref in extra_refs): - raise TargetError(f"{location}.extra_refs contains an invalid release ref") + raise ManifestError(f"{location}.extra_refs contains an invalid release ref") keep_releases = deploy.get("keep_releases", 5) if not isinstance(keep_releases, int) or isinstance(keep_releases, bool) or keep_releases < 1: - raise TargetError(f"{location}.keep_releases must be a positive integer") + raise ManifestError(f"{location}.keep_releases must be a positive integer") hosts = deploy.get("hosts") if not isinstance(hosts, list) or not hosts: - raise TargetError(f"{location}.hosts must be a non-empty array") + raise ManifestError(f"{location}.hosts must be a non-empty array") destinations = [] for destination in hosts: if not isinstance(destination, str) or not SSH_DESTINATION_RE.fullmatch(destination): - raise TargetError(f"{location}.hosts contains an invalid user@host destination") + raise ManifestError(f"{location}.hosts contains an invalid user@host destination") destinations.append(SSH_DESTINATION_RE.fullmatch(destination).group("host")) if len(destinations) != len(set(destinations)): - raise TargetError(f"{location}.hosts contains duplicate host addresses") + raise ManifestError(f"{location}.hosts contains duplicate host addresses") path = require_string(deploy, "path", location, "~/flightdeck") sops_key_file = require_string( deploy, @@ -192,7 +192,7 @@ def load_config(path, mode): try: value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) except yaml.YAMLError as exc: - raise TargetError(f"{path} is not valid YAML: {exc}") from exc + raise ManifestError(f"{path} is not valid YAML: {exc}") from exc config = require_mapping(value, str(path)) if mode == "encrypt": return validate_encrypt(config, str(path)) @@ -201,26 +201,26 @@ def load_config(path, mode): def build_matrix(directory, mode, selected="all"): if mode not in {"encrypt", "deploy"}: - raise TargetError("mode must be encrypt or deploy") + raise ManifestError("mode must be encrypt or deploy") paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) if not paths: - raise TargetError(f"no target manifests found in {directory}") + raise ManifestError(f"no manifests found in {directory}") names = {path.stem for path in paths} if selected != "all" and selected not in names: - raise TargetError(f"unknown target: {selected}") + raise ManifestError(f"unknown name: {selected}") include = [] seen_names = set() for path in paths: name = path.stem - if not TARGET_NAME_RE.fullmatch(name): - raise TargetError(f"invalid target filename: {path.name}") + if not MANIFEST_NAME_RE.fullmatch(name): + raise ManifestError(f"invalid manifest filename: {path.name}") if name in seen_names: - raise TargetError(f"duplicate target name: {name}") + raise ManifestError(f"duplicate manifest name: {name}") seen_names.add(name) config = load_config(path, mode) if selected != "all" and name != selected: continue - item = {"target": name, "manifest": str(path)} + item = {"name": name, "manifest": str(path)} item.update(config) include.append(item) return {"include": include} @@ -237,15 +237,15 @@ def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument("--directory", type=Path) parser.add_argument("--mode", required=True) - parser.add_argument("--target", default="all") + parser.add_argument("--name", default="all") args = parser.parse_args(argv) try: directory = args.directory or Path("encrypt" if args.mode == "encrypt" else "targets") - matrix = build_matrix(directory, args.mode, args.target) + matrix = build_matrix(directory, args.mode, args.name) encoded = json.dumps(matrix, separators=(",", ":")) write_github_output("matrix", encoded) write_github_output("count", len(matrix["include"])) - except TargetError as exc: + except ManifestError as exc: print(f"error: {exc}", file=sys.stderr) return 1 return 0 diff --git a/.github/actions/load-targets/tests/test_load_targets.py b/.github/actions/load-matrix/tests/test_load_matrix.py similarity index 69% rename from .github/actions/load-targets/tests/test_load_targets.py rename to .github/actions/load-matrix/tests/test_load_matrix.py index 1d7eee3..f9dbed0 100644 --- a/.github/actions/load-targets/tests/test_load_targets.py +++ b/.github/actions/load-matrix/tests/test_load_matrix.py @@ -3,10 +3,10 @@ import unittest from pathlib import Path -MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-targets.py" -SPEC = importlib.util.spec_from_file_location("load_targets", MODULE_PATH) -load_targets = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(load_targets) +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-matrix.py" +SPEC = importlib.util.spec_from_file_location("load_matrix", MODULE_PATH) +load_matrix = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(load_matrix) ENCRYPT = """\ @@ -34,7 +34,7 @@ """ -class LoadTargetsTest(unittest.TestCase): +class LoadMatrixTest(unittest.TestCase): def setUp(self): self.temporary_directory = tempfile.TemporaryDirectory() self.addCleanup(self.temporary_directory.cleanup) @@ -47,13 +47,13 @@ def setUp(self): (self.targets_directory / "hawkeye.yml").write_text(TARGET) def test_builds_encrypt_matrix(self): - matrix = load_targets.build_matrix(self.encrypt_directory, "encrypt") + matrix = load_matrix.build_matrix(self.encrypt_directory, "encrypt") self.assertEqual( matrix, { "include": [ { - "target": "hawkeye", + "name": "hawkeye", "manifest": str(self.encrypt_directory / "hawkeye.yml"), "asset": "hawkeye.sops.env", } @@ -62,7 +62,7 @@ def test_builds_encrypt_matrix(self): ) def test_builds_deploy_matrix(self): - item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] + item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) @@ -77,7 +77,7 @@ def test_allows_deploy_path_default_overrides(self): "sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" ) ) - item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] + item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["path"], "~/flightdeck") self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") @@ -87,7 +87,7 @@ def test_allows_deploy_path_default_overrides(self): "sops_age_key_file: ~/.config/sops/age/custom.txt", ) ) - item = load_targets.build_matrix(self.targets_directory, "deploy")["include"][0] + item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] self.assertEqual(item["path"], "~/custom") self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/custom.txt") @@ -95,22 +95,22 @@ def test_filters_selected_config(self): (self.encrypt_directory / "other.yml").write_text( ENCRYPT.replace("hawkeye.sops.env", "other.sops.env") ) - matrix = load_targets.build_matrix(self.encrypt_directory, "encrypt", "hawkeye") - self.assertEqual([item["target"] for item in matrix["include"]], ["hawkeye"]) + matrix = load_matrix.build_matrix(self.encrypt_directory, "encrypt", "hawkeye") + self.assertEqual([item["name"] for item in matrix["include"]], ["hawkeye"]) def test_rejects_unknown_config(self): - with self.assertRaisesRegex(load_targets.TargetError, "unknown target"): - load_targets.build_matrix(self.targets_directory, "deploy", "missing") + with self.assertRaisesRegex(load_matrix.ManifestError, "unknown name"): + load_matrix.build_matrix(self.targets_directory, "deploy", "missing") def test_rejects_wrong_config_type(self): - with self.assertRaisesRegex(load_targets.TargetError, "unknown keys"): - load_targets.build_matrix(self.encrypt_directory, "deploy") + with self.assertRaisesRegex(load_matrix.ManifestError, "unknown keys"): + load_matrix.build_matrix(self.encrypt_directory, "deploy") def test_rejects_invalid_ssh_destination(self): path = self.targets_directory / "hawkeye.yml" path.write_text(TARGET.replace("rubykatzen-com@100.75.50.2", "100.75.50.2")) - with self.assertRaisesRegex(load_targets.TargetError, "invalid user@host destination"): - load_targets.build_matrix(self.targets_directory, "deploy") + with self.assertRaisesRegex(load_matrix.ManifestError, "invalid user@host destination"): + load_matrix.build_matrix(self.targets_directory, "deploy") def test_rejects_duplicate_host_addresses(self): path = self.targets_directory / "hawkeye.yml" @@ -120,26 +120,26 @@ def test_rejects_duplicate_host_addresses(self): "hosts: [first@100.75.50.2, second@100.75.50.2]", ) ) - with self.assertRaisesRegex(load_targets.TargetError, "duplicate host addresses"): - load_targets.build_matrix(self.targets_directory, "deploy") + with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate host addresses"): + load_matrix.build_matrix(self.targets_directory, "deploy") def test_rejects_apps_in_env(self): path = self.encrypt_directory / "hawkeye.yml" path.write_text(ENCRYPT.replace(" APPS_DOMAIN:", " APPS: RUBYKATZEN_COM_APPS\n APPS_DOMAIN:")) - with self.assertRaisesRegex(load_targets.TargetError, "must be configured through"): - load_targets.build_matrix(self.encrypt_directory, "encrypt") + with self.assertRaisesRegex(load_matrix.ManifestError, "must be configured through"): + load_matrix.build_matrix(self.encrypt_directory, "encrypt") def test_rejects_duplicate_apps(self): path = self.encrypt_directory / "hawkeye.yml" path.write_text(ENCRYPT.replace("apps: [traefik, rybbit]", "apps: [traefik, rybbit, traefik]")) - with self.assertRaisesRegex(load_targets.TargetError, "duplicate app names"): - load_targets.build_matrix(self.encrypt_directory, "encrypt") + with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate app names"): + load_matrix.build_matrix(self.encrypt_directory, "encrypt") def test_rejects_duplicate_keys(self): path = self.encrypt_directory / "hawkeye.yml" path.write_text(ENCRYPT.replace("asset: hawkeye.sops.env", "asset: one.sops.env\nasset: two.sops.env")) - with self.assertRaisesRegex(load_targets.TargetError, "duplicate YAML key"): - load_targets.build_matrix(self.encrypt_directory, "encrypt") + with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate YAML key"): + load_matrix.build_matrix(self.encrypt_directory, "encrypt") if __name__ == "__main__": diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4ed261d..1fdf596 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,11 +14,11 @@ jobs: count: ${{ steps.targets.outputs.count }} steps: - uses: actions/checkout@v7 - - uses: $/.github/actions/load-targets + - uses: $/.github/actions/load-matrix id: targets with: mode: deploy - target: ${{ inputs.target || 'all' }} + name: ${{ inputs.target || 'all' }} deploy: needs: targets if: needs.targets.outputs.count != '0' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 801a0ca..266419e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,26 +43,26 @@ jobs: restart.sh up.sh README.md - targets: + encrypt-configs: needs: release if: needs.release.outputs.release_created == 'true' runs-on: ubuntu-latest outputs: - matrix: ${{ steps.targets.outputs.matrix }} - count: ${{ steps.targets.outputs.count }} + matrix: ${{ steps.matrix.outputs.matrix }} + count: ${{ steps.matrix.outputs.count }} steps: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-targets - id: targets + - uses: $/.github/actions/load-matrix + id: matrix with: mode: encrypt encrypt: - needs: [release, targets] - if: needs.targets.outputs.count != '0' + needs: [release, encrypt-configs] + if: needs.encrypt-configs.outputs.count != '0' strategy: - matrix: ${{ fromJson(needs.targets.outputs.matrix) }} + matrix: ${{ fromJson(needs.encrypt-configs.outputs.matrix) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -81,14 +81,14 @@ jobs: if: needs.release.outputs.release_created == 'true' runs-on: ubuntu-latest outputs: - matrix: ${{ steps.targets.outputs.matrix }} - count: ${{ steps.targets.outputs.count }} + matrix: ${{ steps.matrix.outputs.matrix }} + count: ${{ steps.matrix.outputs.count }} steps: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-targets - id: targets + - uses: $/.github/actions/load-matrix + id: matrix with: mode: deploy deploy: diff --git a/README.md b/README.md index bfeb250..d8b19a8 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ flightdeck/ │ ├── actions/ │ │ ├── build-bundle/ # Build and upload a release bundle │ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release -│ │ └── load-targets/ # Validate targets and build workflow matrices +│ │ └── load-matrix/ # Validate configs and build workflow matrices │ └── workflows/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets @@ -441,7 +441,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-targets`) and one reusable workflow, `deploy-shared.yml`. +This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-matrix`) and one reusable workflow, `deploy-shared.yml`. --- @@ -482,7 +482,7 @@ credentials: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-targets` validates each collection and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-matrix` validates each collection and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. --- From b6aad6ac8020cbbe528dc588e3aef2549689f949 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 23:00:02 +0200 Subject: [PATCH 28/35] refactor: make load-matrix a dumb directory-to-matrix reader Rename load-matrix to load-yaml-matrix and strip out its encrypt/deploy schema knowledge. It now just reads every YAML file in a directory, merges each file's top-level fields with name/manifest, and builds a matrix - no mode, no field validation, no defaults. encrypt-env already re-parses and validates its own manifest from the file path, so the encrypt/ path loses nothing. The targets/ path had no other validator, so move what it relied on into the callers: release.yml and deploy.yml now apply the path/keep-releases/sops-age- key-file defaults inline and read credentials.secrets/credentials.variables directly off the raw matrix item instead of the flattened fields load-matrix used to produce. This drops schema validation entirely (SSH destination format, ref format, duplicate hosts, unknown-key typos) - a malformed targets/*.yml now only surfaces once deploy-shared.yml or ansible/deploy.yml actually runs against production hosts, not before. --- .github/actions/load-matrix/README.md | 18 -- .github/actions/load-matrix/action.yml | 40 --- .../load-matrix/scripts/load-matrix.py | 255 ------------------ .../load-matrix/tests/test_load_matrix.py | 146 ---------- .github/actions/load-yaml-matrix/README.md | 17 ++ .github/actions/load-yaml-matrix/action.yml | 31 +++ .../requirements.txt | 0 .../scripts/load-yaml-matrix.py | 97 +++++++ .../tests/test_load_yaml_matrix.py | 81 ++++++ .github/workflows/deploy.yml | 18 +- .github/workflows/release.yml | 22 +- README.md | 8 +- 12 files changed, 251 insertions(+), 482 deletions(-) delete mode 100644 .github/actions/load-matrix/README.md delete mode 100644 .github/actions/load-matrix/action.yml delete mode 100644 .github/actions/load-matrix/scripts/load-matrix.py delete mode 100644 .github/actions/load-matrix/tests/test_load_matrix.py create mode 100644 .github/actions/load-yaml-matrix/README.md create mode 100644 .github/actions/load-yaml-matrix/action.yml rename .github/actions/{load-matrix => load-yaml-matrix}/requirements.txt (100%) create mode 100644 .github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py create mode 100644 .github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py diff --git a/.github/actions/load-matrix/README.md b/.github/actions/load-matrix/README.md deleted file mode 100644 index 5419a85..0000000 --- a/.github/actions/load-matrix/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# load-matrix - -Composite GitHub Action that validates encryption configs or deployment targets and builds a GitHub Actions matrix. - -## Usage - -```yaml -- uses: rubykatzen/flightdeck/.github/actions/load-matrix@main - id: matrix - with: - mode: deploy # encrypt or deploy - name: all # default: all - # directory: custom # optional; defaults to encrypt/ or targets/ based on mode -``` - -The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Encrypt mode validates the flat encryption schema; deploy mode validates the flat target schema. - -Every file is fully validated before it is added to the matrix. Files may use either the `.yml` or `.yaml` extension, and their basename becomes the matrix item's `name`. diff --git a/.github/actions/load-matrix/action.yml b/.github/actions/load-matrix/action.yml deleted file mode 100644 index 86ae25d..0000000 --- a/.github/actions/load-matrix/action.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Load matrix -description: Validate config manifests and build an encrypt or deploy strategy matrix. -inputs: - mode: - description: Configuration type to load (encrypt or deploy). - required: true - name: - description: Single manifest name to load, or all. - required: false - default: all - directory: - description: Directory containing YAML files. Defaults to encrypt for encrypt mode and targets for deploy mode. - required: false - default: "" -outputs: - matrix: - description: JSON strategy matrix containing validated configuration data. - value: ${{ steps.load.outputs.matrix }} - count: - description: Number of targets in the matrix. - value: ${{ steps.load.outputs.count }} -runs: - using: composite - steps: - - name: Install dependencies - shell: bash - run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" - - name: Load matrix - id: load - shell: bash - env: - DIRECTORY: ${{ inputs.directory }} - MODE: ${{ inputs.mode }} - NAME: ${{ inputs.name }} - run: | - directory_args=() - if [ -n "$DIRECTORY" ]; then - directory_args=(--directory "$DIRECTORY") - fi - python3 "$GITHUB_ACTION_PATH/scripts/load-matrix.py" "${directory_args[@]}" --mode "$MODE" --name "$NAME" diff --git a/.github/actions/load-matrix/scripts/load-matrix.py b/.github/actions/load-matrix/scripts/load-matrix.py deleted file mode 100644 index d6128b0..0000000 --- a/.github/actions/load-matrix/scripts/load-matrix.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import re -import sys -from pathlib import Path - -import yaml - -MANIFEST_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") -ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") -SOURCE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:__[A-Z0-9_]+)*$") -KEY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") -APP_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -SSH_DESTINATION_RE = re.compile( - r"^(?P[a-z_][a-z0-9_-]*)@(?P[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?)$" -) -RELEASE_REF_RE = re.compile( - r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$" -) -ENV_REF_RE = re.compile( - r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+\.sops\.env$" -) -ASSET_RE = re.compile(r"^[A-Za-z0-9_.-]+\.sops\.env$") - - -class ManifestError(Exception): - pass - - -class UniqueKeyLoader(yaml.SafeLoader): - pass - - -def construct_mapping(loader, node, deep=False): - mapping = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - if key in mapping: - raise ManifestError(f"duplicate YAML key: {key}") - mapping[key] = loader.construct_object(value_node, deep=deep) - return mapping - - -UniqueKeyLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping, -) - - -def require_mapping(value, location): - if not isinstance(value, dict): - raise ManifestError(f"{location} must be a mapping") - return value - - -def reject_unknown(mapping, allowed, location): - unknown = sorted(set(mapping) - set(allowed)) - if unknown: - raise ManifestError(f"{location} contains unknown keys: {', '.join(unknown)}") - - -def require_string(mapping, key, location, default=None): - value = mapping.get(key, default) - if not isinstance(value, str) or not value: - raise ManifestError(f"{location}.{key} must be a non-empty string") - return value - - -def validate_encrypt(value, location): - encrypt = require_mapping(value, location) - reject_unknown(encrypt, {"asset", "keys", "apps", "env"}, location) - asset = require_string(encrypt, "asset", location) - if not ASSET_RE.fullmatch(asset): - raise ManifestError(f"{location}.asset must be named like server.sops.env") - keys = encrypt.get("keys") - if not isinstance(keys, list) or not keys: - raise ManifestError(f"{location}.keys must be a non-empty array") - if any(not isinstance(key, str) or not KEY_NAME_RE.fullmatch(key) for key in keys): - raise ManifestError(f"{location}.keys contains an invalid age key name") - apps = encrypt.get("apps") - if not isinstance(apps, list) or not apps: - raise ManifestError(f"{location}.apps must be a non-empty array") - if any(not isinstance(app, str) or not APP_NAME_RE.fullmatch(app) for app in apps): - raise ManifestError(f"{location}.apps contains an invalid app name") - if len(apps) != len(set(apps)): - raise ManifestError(f"{location}.apps contains duplicate app names") - env = require_mapping(encrypt.get("env"), f"{location}.env") - if not env: - raise ManifestError(f"{location}.env must not be empty") - for output_name, source_name in env.items(): - if not isinstance(output_name, str) or not ENV_NAME_RE.fullmatch(output_name): - raise ManifestError(f"{location}.env contains an invalid output name: {output_name!r}") - if not isinstance(source_name, str) or not SOURCE_NAME_RE.fullmatch(source_name): - raise ManifestError(f"{location}.env.{output_name} has an invalid source name") - if "APPS" in env: - raise ManifestError(f"{location}.env.APPS must be configured through {location}.apps") - return {"asset": asset} - - -def validate_credentials(value, location): - credentials = require_mapping(value, location) - reject_unknown(credentials, {"variables", "secrets"}, location) - variables = require_mapping(credentials.get("variables", {}), f"{location}.variables") - secrets = require_mapping(credentials.get("secrets"), f"{location}.secrets") - reject_unknown(variables, {"tailscale_oauth_client_id"}, f"{location}.variables") - reject_unknown( - secrets, - {"ssh_private_key", "tailscale_oauth_secret"}, - f"{location}.secrets", - ) - ssh_private_key = require_string(secrets, "ssh_private_key", f"{location}.secrets") - client_id = variables.get("tailscale_oauth_client_id", "") - client_secret = secrets.get("tailscale_oauth_secret", "") - for name, value_name in ( - ("tailscale_oauth_client_id", client_id), - ("tailscale_oauth_secret", client_secret), - ): - if not isinstance(value_name, str): - raise ManifestError(f"{location}.{name} must be a string") - if bool(client_id) != bool(client_secret): - raise ManifestError(f"{location} must configure both Tailscale OAuth names or neither") - return { - "ssh_private_key_secret": ssh_private_key, - "tailscale_oauth_client_id_variable": client_id, - "tailscale_oauth_secret": client_secret, - } - - -def validate_deploy(value, location): - deploy = require_mapping(value, location) - reject_unknown( - deploy, - { - "flightdeck_ref", - "env_ref", - "extra_refs", - "hosts", - "path", - "sops_age_key_file", - "credentials", - "keep_releases", - }, - location, - ) - flightdeck_ref = require_string(deploy, "flightdeck_ref", location) - env_ref = require_string(deploy, "env_ref", location) - if not RELEASE_REF_RE.fullmatch(flightdeck_ref): - raise ManifestError(f"{location}.flightdeck_ref must be in owner/repo@tag format") - if not ENV_REF_RE.fullmatch(env_ref): - raise ManifestError(f"{location}.env_ref must be in owner/repo@tag:asset.sops.env format") - extra_refs = deploy.get("extra_refs", []) - if not isinstance(extra_refs, list): - raise ManifestError(f"{location}.extra_refs must be an array") - if any(not isinstance(ref, str) or not RELEASE_REF_RE.fullmatch(ref) for ref in extra_refs): - raise ManifestError(f"{location}.extra_refs contains an invalid release ref") - keep_releases = deploy.get("keep_releases", 5) - if not isinstance(keep_releases, int) or isinstance(keep_releases, bool) or keep_releases < 1: - raise ManifestError(f"{location}.keep_releases must be a positive integer") - hosts = deploy.get("hosts") - if not isinstance(hosts, list) or not hosts: - raise ManifestError(f"{location}.hosts must be a non-empty array") - destinations = [] - for destination in hosts: - if not isinstance(destination, str) or not SSH_DESTINATION_RE.fullmatch(destination): - raise ManifestError(f"{location}.hosts contains an invalid user@host destination") - destinations.append(SSH_DESTINATION_RE.fullmatch(destination).group("host")) - if len(destinations) != len(set(destinations)): - raise ManifestError(f"{location}.hosts contains duplicate host addresses") - path = require_string(deploy, "path", location, "~/flightdeck") - sops_key_file = require_string( - deploy, - "sops_age_key_file", - location, - "~/.config/sops/age/keys.txt", - ) - result = { - "flightdeck_ref": flightdeck_ref, - "env_ref": env_ref, - "extra_refs": extra_refs, - "hosts": hosts, - "path": path, - "keep_releases": keep_releases, - "sops_age_key_file": sops_key_file, - } - result.update(validate_credentials(deploy.get("credentials"), f"{location}.credentials")) - return result - - -def load_config(path, mode): - try: - value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) - except yaml.YAMLError as exc: - raise ManifestError(f"{path} is not valid YAML: {exc}") from exc - config = require_mapping(value, str(path)) - if mode == "encrypt": - return validate_encrypt(config, str(path)) - return validate_deploy(config, str(path)) - - -def build_matrix(directory, mode, selected="all"): - if mode not in {"encrypt", "deploy"}: - raise ManifestError("mode must be encrypt or deploy") - paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) - if not paths: - raise ManifestError(f"no manifests found in {directory}") - names = {path.stem for path in paths} - if selected != "all" and selected not in names: - raise ManifestError(f"unknown name: {selected}") - include = [] - seen_names = set() - for path in paths: - name = path.stem - if not MANIFEST_NAME_RE.fullmatch(name): - raise ManifestError(f"invalid manifest filename: {path.name}") - if name in seen_names: - raise ManifestError(f"duplicate manifest name: {name}") - seen_names.add(name) - config = load_config(path, mode) - if selected != "all" and name != selected: - continue - item = {"name": name, "manifest": str(path)} - item.update(config) - include.append(item) - return {"include": include} - - -def write_github_output(name, value): - output_path = os.environ.get("GITHUB_OUTPUT") - if output_path: - with open(output_path, "a", encoding="utf-8") as output: - output.write(f"{name}={value}\n") - - -def main(argv=None): - parser = argparse.ArgumentParser() - parser.add_argument("--directory", type=Path) - parser.add_argument("--mode", required=True) - parser.add_argument("--name", default="all") - args = parser.parse_args(argv) - try: - directory = args.directory or Path("encrypt" if args.mode == "encrypt" else "targets") - matrix = build_matrix(directory, args.mode, args.name) - encoded = json.dumps(matrix, separators=(",", ":")) - write_github_output("matrix", encoded) - write_github_output("count", len(matrix["include"])) - except ManifestError as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/actions/load-matrix/tests/test_load_matrix.py b/.github/actions/load-matrix/tests/test_load_matrix.py deleted file mode 100644 index f9dbed0..0000000 --- a/.github/actions/load-matrix/tests/test_load_matrix.py +++ /dev/null @@ -1,146 +0,0 @@ -import importlib.util -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-matrix.py" -SPEC = importlib.util.spec_from_file_location("load_matrix", MODULE_PATH) -load_matrix = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(load_matrix) - - -ENCRYPT = """\ -asset: hawkeye.sops.env -keys: [hawkeye] -apps: [traefik, rybbit] -env: - APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN -""" - -TARGET = """\ -flightdeck_ref: rubykatzen/flightdeck@v1.2.3 -env_ref: rubykatzen/config@v2.0.0:hawkeye.sops.env -extra_refs: - - rubykatzen/apps@v3.0.0:flightdeck-extra.zip -hosts: [rubykatzen-com@100.75.50.2] -path: ~/flightdeck -sops_age_key_file: ~/.config/sops/age/keys.txt -credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET -""" - - -class LoadMatrixTest(unittest.TestCase): - def setUp(self): - self.temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary_directory.cleanup) - self.root = Path(self.temporary_directory.name) - self.encrypt_directory = self.root / "encrypt" - self.targets_directory = self.root / "targets" - self.encrypt_directory.mkdir() - self.targets_directory.mkdir() - (self.encrypt_directory / "hawkeye.yml").write_text(ENCRYPT) - (self.targets_directory / "hawkeye.yml").write_text(TARGET) - - def test_builds_encrypt_matrix(self): - matrix = load_matrix.build_matrix(self.encrypt_directory, "encrypt") - self.assertEqual( - matrix, - { - "include": [ - { - "name": "hawkeye", - "manifest": str(self.encrypt_directory / "hawkeye.yml"), - "asset": "hawkeye.sops.env", - } - ] - }, - ) - - def test_builds_deploy_matrix(self): - item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] - self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) - self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") - self.assertEqual(item["extra_refs"], ["rubykatzen/apps@v3.0.0:flightdeck-extra.zip"]) - self.assertEqual(item["path"], "~/flightdeck") - self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") - self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") - - def test_allows_deploy_path_default_overrides(self): - path = self.targets_directory / "hawkeye.yml" - path.write_text( - TARGET.replace("path: ~/flightdeck\n", "").replace( - "sops_age_key_file: ~/.config/sops/age/keys.txt\n", "" - ) - ) - item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] - self.assertEqual(item["path"], "~/flightdeck") - self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/keys.txt") - - path.write_text( - TARGET.replace("path: ~/flightdeck", "path: ~/custom").replace( - "sops_age_key_file: ~/.config/sops/age/keys.txt", - "sops_age_key_file: ~/.config/sops/age/custom.txt", - ) - ) - item = load_matrix.build_matrix(self.targets_directory, "deploy")["include"][0] - self.assertEqual(item["path"], "~/custom") - self.assertEqual(item["sops_age_key_file"], "~/.config/sops/age/custom.txt") - - def test_filters_selected_config(self): - (self.encrypt_directory / "other.yml").write_text( - ENCRYPT.replace("hawkeye.sops.env", "other.sops.env") - ) - matrix = load_matrix.build_matrix(self.encrypt_directory, "encrypt", "hawkeye") - self.assertEqual([item["name"] for item in matrix["include"]], ["hawkeye"]) - - def test_rejects_unknown_config(self): - with self.assertRaisesRegex(load_matrix.ManifestError, "unknown name"): - load_matrix.build_matrix(self.targets_directory, "deploy", "missing") - - def test_rejects_wrong_config_type(self): - with self.assertRaisesRegex(load_matrix.ManifestError, "unknown keys"): - load_matrix.build_matrix(self.encrypt_directory, "deploy") - - def test_rejects_invalid_ssh_destination(self): - path = self.targets_directory / "hawkeye.yml" - path.write_text(TARGET.replace("rubykatzen-com@100.75.50.2", "100.75.50.2")) - with self.assertRaisesRegex(load_matrix.ManifestError, "invalid user@host destination"): - load_matrix.build_matrix(self.targets_directory, "deploy") - - def test_rejects_duplicate_host_addresses(self): - path = self.targets_directory / "hawkeye.yml" - path.write_text( - TARGET.replace( - "hosts: [rubykatzen-com@100.75.50.2]", - "hosts: [first@100.75.50.2, second@100.75.50.2]", - ) - ) - with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate host addresses"): - load_matrix.build_matrix(self.targets_directory, "deploy") - - def test_rejects_apps_in_env(self): - path = self.encrypt_directory / "hawkeye.yml" - path.write_text(ENCRYPT.replace(" APPS_DOMAIN:", " APPS: RUBYKATZEN_COM_APPS\n APPS_DOMAIN:")) - with self.assertRaisesRegex(load_matrix.ManifestError, "must be configured through"): - load_matrix.build_matrix(self.encrypt_directory, "encrypt") - - def test_rejects_duplicate_apps(self): - path = self.encrypt_directory / "hawkeye.yml" - path.write_text(ENCRYPT.replace("apps: [traefik, rybbit]", "apps: [traefik, rybbit, traefik]")) - with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate app names"): - load_matrix.build_matrix(self.encrypt_directory, "encrypt") - - def test_rejects_duplicate_keys(self): - path = self.encrypt_directory / "hawkeye.yml" - path.write_text(ENCRYPT.replace("asset: hawkeye.sops.env", "asset: one.sops.env\nasset: two.sops.env")) - with self.assertRaisesRegex(load_matrix.ManifestError, "duplicate YAML key"): - load_matrix.build_matrix(self.encrypt_directory, "encrypt") - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/actions/load-yaml-matrix/README.md b/.github/actions/load-yaml-matrix/README.md new file mode 100644 index 0000000..7e86bd6 --- /dev/null +++ b/.github/actions/load-yaml-matrix/README.md @@ -0,0 +1,17 @@ +# load-yaml-matrix + +Composite GitHub Action that reads every YAML file in a directory into a GitHub Actions matrix. It does no schema validation — callers are responsible for the shape of their own manifests. + +## Usage + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/load-yaml-matrix@main + id: matrix + with: + directory: targets # required + # name: all # optional; single manifest name to load, default: all +``` + +The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Each matrix item merges the manifest's own top-level YAML fields with `name` (the file's basename) and `manifest` (its path). + +Files may use either the `.yml` or `.yaml` extension. Filenames must match `^[a-z0-9][a-z0-9-]*$` and be unique per directory; duplicate top-level YAML keys within a manifest are rejected. Beyond that, the parsed YAML mapping is passed through as-is — validate anything else downstream. diff --git a/.github/actions/load-yaml-matrix/action.yml b/.github/actions/load-yaml-matrix/action.yml new file mode 100644 index 0000000..0b1f6a5 --- /dev/null +++ b/.github/actions/load-yaml-matrix/action.yml @@ -0,0 +1,31 @@ +name: Load YAML matrix +description: Read every YAML file in a directory into a GitHub Actions matrix. No schema validation. +inputs: + directory: + description: Directory containing YAML files. + required: true + name: + description: Single manifest name to load, or all. + required: false + default: all +outputs: + matrix: + description: JSON strategy matrix, one item per manifest, merging its parsed YAML fields with name and manifest. + value: ${{ steps.load.outputs.matrix }} + count: + description: Number of manifests in the matrix. + value: ${{ steps.load.outputs.count }} +runs: + using: composite + steps: + - name: Install dependencies + shell: bash + run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" + - name: Load matrix + id: load + shell: bash + env: + DIRECTORY: ${{ inputs.directory }} + NAME: ${{ inputs.name }} + run: | + python3 "$GITHUB_ACTION_PATH/scripts/load-yaml-matrix.py" --directory "$DIRECTORY" --name "$NAME" diff --git a/.github/actions/load-matrix/requirements.txt b/.github/actions/load-yaml-matrix/requirements.txt similarity index 100% rename from .github/actions/load-matrix/requirements.txt rename to .github/actions/load-yaml-matrix/requirements.txt diff --git a/.github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py b/.github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py new file mode 100644 index 0000000..de13c58 --- /dev/null +++ b/.github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import sys +from pathlib import Path + +import yaml + +NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") + + +class ManifestError(Exception): + pass + + +class UniqueKeyLoader(yaml.SafeLoader): + pass + + +def construct_mapping(loader, node, deep=False): + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise ManifestError(f"duplicate YAML key: {key}") + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping, +) + + +def load_manifest(path): + try: + value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) + except yaml.YAMLError as exc: + raise ManifestError(f"{path} is not valid YAML: {exc}") from exc + if not isinstance(value, dict): + raise ManifestError(f"{path} must contain a YAML mapping") + return value + + +def build_matrix(directory, selected="all"): + paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) + if not paths: + raise ManifestError(f"no manifests found in {directory}") + names = {path.stem for path in paths} + if selected != "all" and selected not in names: + raise ManifestError(f"unknown name: {selected}") + include = [] + seen_names = set() + for path in paths: + name = path.stem + if not NAME_RE.fullmatch(name): + raise ManifestError(f"invalid manifest filename: {path.name}") + if name in seen_names: + raise ManifestError(f"duplicate manifest name: {name}") + seen_names.add(name) + manifest = load_manifest(path) + if selected != "all" and name != selected: + continue + item = {"name": name, "manifest": str(path)} + item.update(manifest) + include.append(item) + return {"include": include} + + +def write_github_output(name, value): + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + output.write(f"{name}={value}\n") + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--directory", required=True, type=Path) + parser.add_argument("--name", default="all") + args = parser.parse_args(argv) + try: + matrix = build_matrix(args.directory, args.name) + encoded = json.dumps(matrix, separators=(",", ":")) + write_github_output("matrix", encoded) + write_github_output("count", len(matrix["include"])) + except ManifestError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py b/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py new file mode 100644 index 0000000..8d1cfb2 --- /dev/null +++ b/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py @@ -0,0 +1,81 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-yaml-matrix.py" +SPEC = importlib.util.spec_from_file_location("load_yaml_matrix", MODULE_PATH) +load_yaml_matrix = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(load_yaml_matrix) + + +HAWKEYE = """\ +flightdeck_ref: rubykatzen/flightdeck@v1.2.3 +hosts: [rubykatzen-com@100.75.50.2] +""" + +MAINFRAME = """\ +flightdeck_ref: rubykatzen/flightdeck@v1.0.0 +hosts: [deploy@100.64.0.1] +""" + + +class LoadYamlMatrixTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.directory = Path(self.temporary_directory.name) + (self.directory / "hawkeye.yml").write_text(HAWKEYE) + (self.directory / "mainframe.yml").write_text(MAINFRAME) + + def test_builds_matrix_from_all_manifests(self): + matrix = load_yaml_matrix.build_matrix(self.directory) + self.assertEqual( + sorted(item["name"] for item in matrix["include"]), + ["hawkeye", "mainframe"], + ) + + def test_merges_manifest_fields_with_name_and_manifest(self): + item = load_yaml_matrix.build_matrix(self.directory, "hawkeye")["include"][0] + self.assertEqual(item["name"], "hawkeye") + self.assertEqual(item["manifest"], str(self.directory / "hawkeye.yml")) + self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") + self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) + + def test_filters_selected_manifest(self): + matrix = load_yaml_matrix.build_matrix(self.directory, "hawkeye") + self.assertEqual([item["name"] for item in matrix["include"]], ["hawkeye"]) + + def test_rejects_unknown_name(self): + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "unknown name"): + load_yaml_matrix.build_matrix(self.directory, "missing") + + def test_rejects_empty_directory(self): + empty = self.directory / "empty" + empty.mkdir() + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "no manifests found"): + load_yaml_matrix.build_matrix(empty) + + def test_rejects_invalid_manifest_filename(self): + (self.directory / "Hawkeye_Prod.yml").write_text(HAWKEYE) + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "invalid manifest filename"): + load_yaml_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_manifest_name(self): + (self.directory / "hawkeye.yaml").write_text(HAWKEYE) + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "duplicate manifest name"): + load_yaml_matrix.build_matrix(self.directory) + + def test_rejects_non_mapping_manifest(self): + (self.directory / "hawkeye.yml").write_text("- one\n- two\n") + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "must contain a YAML mapping"): + load_yaml_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_yaml_key(self): + (self.directory / "hawkeye.yml").write_text("hosts: [one]\nhosts: [two]\n") + with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "duplicate YAML key"): + load_yaml_matrix.build_matrix(self.directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1fdf596..83f7559 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,10 +14,10 @@ jobs: count: ${{ steps.targets.outputs.count }} steps: - uses: actions/checkout@v7 - - uses: $/.github/actions/load-matrix + - uses: $/.github/actions/load-yaml-matrix id: targets with: - mode: deploy + directory: targets name: ${{ inputs.target || 'all' }} deploy: needs: targets @@ -29,11 +29,11 @@ jobs: hosts: ${{ toJson(matrix.hosts) }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} - extra-refs: ${{ toJson(matrix.extra_refs) }} - path: ${{ matrix.path }} - keep-releases: ${{ matrix.keep_releases }} - sops-age-key-file: ${{ matrix.sops_age_key_file }} - tailscale-oauth-client-id: ${{ vars[matrix.tailscale_oauth_client_id_variable] }} + extra-refs: ${{ matrix.extra_refs && toJson(matrix.extra_refs) || '[]' }} + path: ${{ matrix.path || '~/flightdeck' }} + keep-releases: ${{ matrix.keep_releases || 5 }} + sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} + tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} secrets: - ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} - tailscale-oauth-secret: ${{ secrets[matrix.tailscale_oauth_secret] }} + ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} + tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 266419e..8752ca8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,10 +54,10 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-matrix + - uses: $/.github/actions/load-yaml-matrix id: matrix with: - mode: encrypt + directory: encrypt encrypt: needs: [release, encrypt-configs] if: needs.encrypt-configs.outputs.count != '0' @@ -87,10 +87,10 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-matrix + - uses: $/.github/actions/load-yaml-matrix id: matrix with: - mode: deploy + directory: targets deploy: needs: [upload, encrypt, deploy-targets] if: needs.deploy-targets.outputs.count != '0' @@ -101,11 +101,11 @@ jobs: hosts: ${{ toJson(matrix.hosts) }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} - extra-refs: ${{ toJson(matrix.extra_refs) }} - path: ${{ matrix.path }} - keep-releases: ${{ matrix.keep_releases }} - sops-age-key-file: ${{ matrix.sops_age_key_file }} - tailscale-oauth-client-id: ${{ vars[matrix.tailscale_oauth_client_id_variable] }} + extra-refs: ${{ matrix.extra_refs && toJson(matrix.extra_refs) || '[]' }} + path: ${{ matrix.path || '~/flightdeck' }} + keep-releases: ${{ matrix.keep_releases || 5 }} + sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} + tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} secrets: - ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} - tailscale-oauth-secret: ${{ secrets[matrix.tailscale_oauth_secret] }} + ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} + tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} diff --git a/README.md b/README.md index d8b19a8..e58e4ea 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ flightdeck/ │ ├── actions/ │ │ ├── build-bundle/ # Build and upload a release bundle │ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release -│ │ └── load-matrix/ # Validate configs and build workflow matrices +│ │ └── load-yaml-matrix/ # Read a directory of YAML manifests into a workflow matrix │ └── workflows/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets @@ -441,7 +441,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-matrix`) and one reusable workflow, `deploy-shared.yml`. +This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-yaml-matrix`) and one reusable workflow, `deploy-shared.yml`. --- @@ -482,7 +482,9 @@ credentials: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `load-matrix` validates each collection and emits an `encrypt` or `deploy` strategy matrix for the repository workflows. +Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. + +`load-yaml-matrix` reads every file in `encrypt/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. --- From 18a13a0b9d5e15d0c393480de9068298415580e3 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 23:06:03 +0200 Subject: [PATCH 29/35] refactor: rename encrypt/ to envs/ targets/ is a plural noun naming what it holds (deploy targets). encrypt/ was a singular verb, inconsistent with that pattern and with its own contents (env asset configs, one per encrypt-env manifest). Rename to envs/ to match. --- .github/actions/encrypt-env/README.md | 2 +- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- README.md | 12 ++++++------ {encrypt => envs}/hawkeye.yml | 0 5 files changed, 9 insertions(+), 9 deletions(-) rename {encrypt => envs}/hawkeye.yml (100%) diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md index ec9214e..0e159dc 100644 --- a/.github/actions/encrypt-env/README.md +++ b/.github/actions/encrypt-env/README.md @@ -9,7 +9,7 @@ The release must exist before this action runs. ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: encrypt/mainframe.yml # required + manifest: envs/mainframe.yml # required keys-directory: keys # default: keys release-tag: ${{ needs.release.outputs.tag }} # required, must already exist release-repo: "" # default: current repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8752ca8..96a8edf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,7 @@ jobs: - uses: $/.github/actions/load-yaml-matrix id: matrix with: - directory: encrypt + directory: envs encrypt: needs: [release, encrypt-configs] if: needs.encrypt-configs.outputs.count != '0' diff --git a/AGENTS.md b/AGENTS.md index 4f5584b..7f922b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,7 +409,7 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release Deployment helpers live in this repository: - `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` -- `.github/actions/encrypt-env/` is a local composite action for rendering `encrypt/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset +- `.github/actions/encrypt-env/` is a local composite action for rendering `envs/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. diff --git a/README.md b/README.md index e58e4ea..9b6bd8c 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ flightdeck/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ -├── encrypt/ # Encrypted env asset configurations +├── envs/ # Encrypted env asset configurations ├── targets/ # Deployment targets ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template @@ -445,11 +445,11 @@ This repository provides three composite actions under `.github/actions/` (`buil --- -### Encrypt Configurations And Targets +### Envs And Targets -Files in `encrypt/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. +Files in `envs/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. -`encrypt/mainframe.yml`: +`envs/mainframe.yml`: ```yaml asset: mainframe.sops.env @@ -484,7 +484,7 @@ credentials: Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. -`load-yaml-matrix` reads every file in `encrypt/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. +`load-yaml-matrix` reads every file in `envs/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. --- @@ -495,7 +495,7 @@ Renders an encryption config from GitHub Secrets/Variables, encrypts it with SOP ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: encrypt/mainframe.yml # required + manifest: envs/mainframe.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository diff --git a/encrypt/hawkeye.yml b/envs/hawkeye.yml similarity index 100% rename from encrypt/hawkeye.yml rename to envs/hawkeye.yml From 9a419a3665b3beb3a05f6862c221cdbea16d985f Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 23:10:20 +0200 Subject: [PATCH 30/35] refactor: rename envs/ to sops/ envs/ read unnaturally next to targets/ and keys/. sops/ matches the asset naming convention already used throughout (*.sops.env) and pairs directly with keys/, which already holds the age recipients used to encrypt into that directory's assets. --- .github/actions/encrypt-env/README.md | 2 +- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- README.md | 12 ++++++------ {envs => sops}/hawkeye.yml | 0 5 files changed, 9 insertions(+), 9 deletions(-) rename {envs => sops}/hawkeye.yml (100%) diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md index 0e159dc..ac57b93 100644 --- a/.github/actions/encrypt-env/README.md +++ b/.github/actions/encrypt-env/README.md @@ -9,7 +9,7 @@ The release must exist before this action runs. ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: envs/mainframe.yml # required + manifest: sops/mainframe.yml # required keys-directory: keys # default: keys release-tag: ${{ needs.release.outputs.tag }} # required, must already exist release-repo: "" # default: current repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96a8edf..784048d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,7 @@ jobs: - uses: $/.github/actions/load-yaml-matrix id: matrix with: - directory: envs + directory: sops encrypt: needs: [release, encrypt-configs] if: needs.encrypt-configs.outputs.count != '0' diff --git a/AGENTS.md b/AGENTS.md index 7f922b1..89ce2a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,7 +409,7 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release Deployment helpers live in this repository: - `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` -- `.github/actions/encrypt-env/` is a local composite action for rendering `envs/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset +- `.github/actions/encrypt-env/` is a local composite action for rendering `sops/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. diff --git a/README.md b/README.md index 9b6bd8c..5a5186e 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ flightdeck/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ -├── envs/ # Encrypted env asset configurations +├── sops/ # Encrypted env asset configurations ├── targets/ # Deployment targets ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template @@ -445,11 +445,11 @@ This repository provides three composite actions under `.github/actions/` (`buil --- -### Envs And Targets +### Sops And Targets -Files in `envs/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. +Files in `sops/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. -`envs/mainframe.yml`: +`sops/mainframe.yml`: ```yaml asset: mainframe.sops.env @@ -484,7 +484,7 @@ credentials: Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. -`load-yaml-matrix` reads every file in `envs/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. +`load-yaml-matrix` reads every file in `sops/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. --- @@ -495,7 +495,7 @@ Renders an encryption config from GitHub Secrets/Variables, encrypts it with SOP ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: envs/mainframe.yml # required + manifest: sops/mainframe.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository diff --git a/envs/hawkeye.yml b/sops/hawkeye.yml similarity index 100% rename from envs/hawkeye.yml rename to sops/hawkeye.yml From 9ba35e3415d63811cd0a18c0b535de068e3af533 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 23:31:30 +0200 Subject: [PATCH 31/35] refactor: rename sops/ to vaults/ One repo prepares vaults from a pool of secrets/vars; different consumers (targets) unpack the vault meant for them. Also just a better English word for "a named container of credentials" than sops/, which only referenced the encryption mechanism. --- .github/actions/encrypt-env/README.md | 2 +- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- README.md | 12 ++++++------ {sops => vaults}/hawkeye.yml | 0 5 files changed, 9 insertions(+), 9 deletions(-) rename {sops => vaults}/hawkeye.yml (100%) diff --git a/.github/actions/encrypt-env/README.md b/.github/actions/encrypt-env/README.md index ac57b93..b02292b 100644 --- a/.github/actions/encrypt-env/README.md +++ b/.github/actions/encrypt-env/README.md @@ -9,7 +9,7 @@ The release must exist before this action runs. ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: sops/mainframe.yml # required + manifest: vaults/mainframe.yml # required keys-directory: keys # default: keys release-tag: ${{ needs.release.outputs.tag }} # required, must already exist release-repo: "" # default: current repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 784048d..357017b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,7 @@ jobs: - uses: $/.github/actions/load-yaml-matrix id: matrix with: - directory: sops + directory: vaults encrypt: needs: [release, encrypt-configs] if: needs.encrypt-configs.outputs.count != '0' diff --git a/AGENTS.md b/AGENTS.md index 89ce2a6..7ff448c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,7 +409,7 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release Deployment helpers live in this repository: - `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` -- `.github/actions/encrypt-env/` is a local composite action for rendering `sops/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset +- `.github/actions/encrypt-env/` is a local composite action for rendering `vaults/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. diff --git a/README.md b/README.md index 5a5186e..1c1fe6c 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ flightdeck/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ -├── sops/ # Encrypted env asset configurations +├── vaults/ # Encrypted env asset configurations ├── targets/ # Deployment targets ├── .env # All server configuration incl. APPS list (git-ignored) ├── .env.example # Configuration template @@ -445,11 +445,11 @@ This repository provides three composite actions under `.github/actions/` (`buil --- -### Sops And Targets +### Vaults And Targets -Files in `sops/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. +Files in `vaults/` describe encrypted env assets. Files in `targets/` describe deployments. The two collections are independent; a deployment links to an encrypted asset explicitly through `env_ref`. Matching filenames are a convenience, not an implicit relationship. -`sops/mainframe.yml`: +`vaults/mainframe.yml`: ```yaml asset: mainframe.sops.env @@ -484,7 +484,7 @@ credentials: Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. -`load-yaml-matrix` reads every file in `sops/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. +`load-yaml-matrix` reads every file in `vaults/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. --- @@ -495,7 +495,7 @@ Renders an encryption config from GitHub Secrets/Variables, encrypts it with SOP ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: sops/mainframe.yml # required + manifest: vaults/mainframe.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository diff --git a/sops/hawkeye.yml b/vaults/hawkeye.yml similarity index 100% rename from sops/hawkeye.yml rename to vaults/hawkeye.yml From ae6f6a10e752f6ad1bddb95a0b140dd16b8cc3d4 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Wed, 19 Aug 2026 23:37:55 +0200 Subject: [PATCH 32/35] refactor: homogenize matrix job naming across release.yml and deploy.yml encrypt-configs -> encrypt-vaults, matching the deploy-targets job it sits next to (both - now). deploy.yml's targets job becomes deploy-targets to match release.yml's job of the same purpose, and its step id follows the matrix convention used everywhere else. --- .github/workflows/deploy.yml | 14 +++++++------- .github/workflows/release.yml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 83f7559..29e1b6d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,23 +7,23 @@ on: type: string default: all jobs: - targets: + deploy-targets: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.targets.outputs.matrix }} - count: ${{ steps.targets.outputs.count }} + matrix: ${{ steps.matrix.outputs.matrix }} + count: ${{ steps.matrix.outputs.count }} steps: - uses: actions/checkout@v7 - uses: $/.github/actions/load-yaml-matrix - id: targets + id: matrix with: directory: targets name: ${{ inputs.target || 'all' }} deploy: - needs: targets - if: needs.targets.outputs.count != '0' + needs: deploy-targets + if: needs.deploy-targets.outputs.count != '0' strategy: - matrix: ${{ fromJson(needs.targets.outputs.matrix) }} + matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} uses: $/.github/workflows/deploy-shared.yml with: hosts: ${{ toJson(matrix.hosts) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 357017b..423eb86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,7 @@ jobs: restart.sh up.sh README.md - encrypt-configs: + encrypt-vaults: needs: release if: needs.release.outputs.release_created == 'true' runs-on: ubuntu-latest @@ -59,10 +59,10 @@ jobs: with: directory: vaults encrypt: - needs: [release, encrypt-configs] - if: needs.encrypt-configs.outputs.count != '0' + needs: [release, encrypt-vaults] + if: needs.encrypt-vaults.outputs.count != '0' strategy: - matrix: ${{ fromJson(needs.encrypt-configs.outputs.matrix) }} + matrix: ${{ fromJson(needs.encrypt-vaults.outputs.matrix) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 From 285b1e1161821789b4ea6c29135983e6565c2678 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Thu, 20 Aug 2026 00:05:10 +0200 Subject: [PATCH 33/35] refactor: publish flightdeck's apps/ as its own release asset Core apps and third-party extra apps were structurally identical (an apps/ directory) but packaged asymmetrically: core was baked into flightdeck.zip, extras went through a separate build-bundle+merge path. Publish flightdeck's own apps/ as flightdeck-apps.zip via the same build-bundle action, and merge it through the exact same ansible path as any other app bundle - removing the special-cased "extract core zip as the apps base" step entirely. extra_refs is retired in favor of app_refs, now required and non-empty throughout (targets/*.yml, deploy-shared.yml's app-refs input, and ansible's flightdeck_app_refs): a target must explicitly list every app bundle it wants, including flightdeck's own - there's no more implicit apps/ riding along with the machinery bundle. --- .github/workflows/deploy-shared.yml | 14 +++++------ .github/workflows/deploy.yml | 2 +- .github/workflows/release.yml | 19 ++++++++++++--- AGENTS.md | 6 ++--- README.md | 26 ++++++++++---------- ansible/deploy.yml | 37 ++++++++++++++++------------- targets/hawkeye.yml | 3 ++- 7 files changed, 63 insertions(+), 44 deletions(-) diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 7bdf79a..5f58bed 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -14,10 +14,10 @@ on: description: Release ref for the encrypted env package, in owner/repo@tag:asset format. type: string required: true - extra-refs: - description: JSON array of release refs for optional extra app bundles. + app-refs: + description: JSON array of release refs for app bundles to merge into the release, in owner/repo@tag[:asset] format. type: string - default: "[]" + required: true path: description: Base path on the target host for releases, shared files, and the current symlink. type: string @@ -75,14 +75,14 @@ jobs: env: APP_REF: ${{ inputs.app-ref }} ENV_REF: ${{ inputs.env-ref }} - EXTRA_REFS: ${{ inputs.extra-refs }} + APP_REFS: ${{ inputs.app-refs }} HOSTS: ${{ inputs.hosts }} DEPLOY_PATH: ${{ inputs.path }} KEEP_RELEASES: ${{ inputs.keep-releases }} SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} run: | hosts_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and test("^[a-z_][a-z0-9_-]*@[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$")) then . else error("hosts must be a non-empty user@host string array") end' <<< "$HOSTS")" - extra_refs_json="$(jq -ce 'if type == "array" and all(.[]; type == "string" and length > 0) then . else error("extra-refs must be a string array") end' <<< "$EXTRA_REFS")" + app_refs_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("app-refs must be a non-empty string array") end' <<< "$APP_REFS")" jq -ce ' reduce .[] as $destination ({all: {hosts: {}}}; ($destination | capture("^(?[^@]+)@(?.+)$")) as $ssh | @@ -92,11 +92,11 @@ jobs: json="$(jq -n \ --arg app_ref "$APP_REF" \ --arg env_ref "$ENV_REF" \ - --argjson extra_refs "$extra_refs_json" \ + --argjson app_refs "$app_refs_json" \ --arg path "$DEPLOY_PATH" \ --argjson keep_releases "$KEEP_RELEASES" \ --arg sops_key_file "$SOPS_KEY_FILE" \ - '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_extra_refs: $extra_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" + '{flightdeck_app_ref: $app_ref, flightdeck_env_ref: $env_ref, flightdeck_app_refs: $app_refs, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" echo "json=$json" >> "$GITHUB_OUTPUT" echo "inventory=$RUNNER_TEMP/flightdeck-inventory.json" >> "$GITHUB_OUTPUT" - name: Run playbook diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 29e1b6d..b2034d8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,7 +29,7 @@ jobs: hosts: ${{ toJson(matrix.hosts) }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} - extra-refs: ${{ matrix.extra_refs && toJson(matrix.extra_refs) || '[]' }} + app-refs: ${{ toJson(matrix.app_refs) }} path: ${{ matrix.path || '~/flightdeck' }} keep-releases: ${{ matrix.keep_releases || 5 }} sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 423eb86..aad278b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,6 @@ jobs: release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} paths: | - apps ansible.cfg .env.example backup.sh @@ -43,6 +42,20 @@ jobs: restart.sh up.sh README.md + upload-apps: + needs: release + if: needs.release.outputs.release_created == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.release.outputs.tag_name }} + - uses: $/.github/actions/build-bundle + with: + bundle-name: flightdeck-apps.zip + release-tag: ${{ needs.release.outputs.tag_name }} + token: ${{ secrets.GITHUB_TOKEN }} + paths: apps encrypt-vaults: needs: release if: needs.release.outputs.release_created == 'true' @@ -92,7 +105,7 @@ jobs: with: directory: targets deploy: - needs: [upload, encrypt, deploy-targets] + needs: [upload, upload-apps, encrypt, deploy-targets] if: needs.deploy-targets.outputs.count != '0' strategy: matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} @@ -101,7 +114,7 @@ jobs: hosts: ${{ toJson(matrix.hosts) }} app-ref: ${{ matrix.flightdeck_ref }} env-ref: ${{ matrix.env_ref }} - extra-refs: ${{ matrix.extra_refs && toJson(matrix.extra_refs) || '[]' }} + app-refs: ${{ toJson(matrix.app_refs) }} path: ${{ matrix.path || '~/flightdeck' }} keep-releases: ${{ matrix.keep_releases || 5 }} sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} diff --git a/AGENTS.md b/AGENTS.md index 7ff448c..7547fda 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -404,15 +404,15 @@ GitHub Actions workflow (`.github/workflows/release-please.yml`) manages release 1. On every push to `main`, Release Please opens/updates a `chore(main): release X.Y.Z` PR with the computed version and generated `CHANGELOG.md` entry 2. Merging that PR tags the release and publishes a GitHub Release -3. A second job then builds `flightdeck.zip` from compose files, helper scripts, examples, and README (verifying runtime state such as `.env`, `apps-data`, `backups`, and generated `apps/*/.env` is excluded) and uploads it as a release asset. Deploy refs may use `@latest` as a playbook-side alias for GitHub's latest release API; no mutable `latest` release/tag is created. +3. A second and third job then build and upload two release assets: `flightdeck.zip` from helper scripts, examples, and README (verifying runtime state such as `.env`, `apps-data`, `backups`, and generated `apps/*/.env` is excluded), and `flightdeck-apps.zip` from the `apps/` catalog alone. Deploy refs may use `@latest` as a playbook-side alias for GitHub's latest release API; no mutable `latest` release/tag is created. Deployment helpers live in this repository: -- `ansible/deploy.yml` pulls `flightdeck_app_ref`, merges optional `flightdeck_extra_refs`, pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` +- `ansible/deploy.yml` pulls `flightdeck_app_ref` (the machinery bundle), merges every ref in `flightdeck_app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit), pulls the server-specific encrypted env package from `flightdeck_env_ref`, decrypts `.sops.env` on the server, switches a timestamped release, and runs `./deploy.sh` - `.github/actions/encrypt-env/` is a local composite action for rendering `vaults/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository -Extra Flightdeck bundles are release assets referenced as short refs like `/@latest` or `/@v1.2.3`. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Extra bundles must contain an `apps/` directory only adding app directories; app names may not conflict with the core bundle or earlier extras. +App bundles listed in `flightdeck_app_refs` are release assets referenced as short refs like `/@latest` or `/@v1.2.3`, resolving to a default asset name of `flightdeck-apps.zip` unless the ref specifies an explicit `:asset-name` suffix. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Every bundle must contain an `apps/` directory; app names may not conflict across bundles. Private release assets are supported by passing `FLIGHTDECK_GITHUB_TOKEN` as a secret environment variable to `ansible/deploy.yml`. Store it as a secret in whatever system runs the playbook (e.g. a GitHub Actions secret when using `deploy-shared.yml`), not in plain configuration. When the token is present, the playbook exports it as `GH_TOKEN` for `gh release download`. diff --git a/README.md b/README.md index 1c1fe6c..cfbde27 100644 --- a/README.md +++ b/README.md @@ -90,12 +90,13 @@ Target servers need Docker, Docker Compose, GitHub CLI (`gh`), SOPS, and the ser ansible-playbook ansible/deploy.yml \ -i mainframe, \ -u root \ - -e flightdeck_env_ref=/@latest:.sops.env + -e flightdeck_env_ref=/@latest:.sops.env \ + -e '{"flightdeck_app_refs":["rubykatzen/flightdeck@latest"]}' ``` The `flightdeck_env_ref` format is `owner/repo@tag:asset`. Use an immutable semver tag for a pinned deploy, or `@latest` to resolve GitHub's latest release at deploy time. The playbook downloads the asset, decrypts it with the server-local SOPS age key (`flightdeck_sops_age_key_file`), links shared `.env` and `apps-data` into a timestamped release, switches `current`, and runs `./deploy.sh`. -The `flightdeck_app_ref` defaults to `rubykatzen/flightdeck@latest`. +`flightdeck_app_ref` (the machinery bundle) and `flightdeck_app_refs` (the app bundles to merge, `owner/repo@tag[:asset]` each) are both required — there's no default and no implicit `apps/`. `flightdeck_app_refs` must list at least one ref; for the default catalog, that's `rubykatzen/flightdeck@latest`. For private GitHub Releases, pass a token through the `FLIGHTDECK_GITHUB_TOKEN` environment variable. Store it as a secret in whatever system runs this @@ -109,17 +110,17 @@ FLIGHTDECK_GITHUB_TOKEN=... When `FLIGHTDECK_GITHUB_TOKEN` is set, the playbook exports it as `GH_TOKEN` for `gh release download`. Public releases do not need this variable. -Optional extra app bundles can be merged into the release before deploy: +Additional app bundles merge in the exact same way, as further entries in `flightdeck_app_refs`: ```bash ansible-playbook ansible/deploy.yml \ -i mainframe, \ -u root \ -e flightdeck_env_ref=/@latest:.sops.env \ - -e '{"flightdeck_extra_refs":["/@latest"]}' + -e '{"flightdeck_app_refs":["rubykatzen/flightdeck@latest","/@latest"]}' ``` -Extra bundles must contain an `apps/` directory. Extra app names cannot conflict with apps from the core bundle or earlier extra bundles. +Every bundle in `flightdeck_app_refs` must contain an `apps/` directory. App names cannot conflict across bundles. ### 3. Select Applications @@ -257,7 +258,7 @@ The script stops each app one at a time, creates a zip archive, restarts it, the | **databasus** | Database management UI | | **rybbit** | Web analytics | -Additional apps can live in an optional extra catalog repo (`apps/` directory) and be merged at deploy time with `flightdeck_extra_refs`. +This catalog is itself published as its own release asset (`flightdeck-apps.zip`), merged at deploy time like any other entry in `flightdeck_app_refs`. Additional apps can live in any other repo's own `apps/`-shaped catalog, published the same way, and merged in by listing its ref alongside flightdeck's own. ## ⚙️ Configuration @@ -467,7 +468,8 @@ env: ```yaml flightdeck_ref: rubykatzen/flightdeck@latest env_ref: owner/config@latest:mainframe.sops.env -extra_refs: +app_refs: + - rubykatzen/flightdeck@latest - owner/extra-apps@latest hosts: - deploy@100.64.0.1 @@ -482,7 +484,7 @@ credentials: tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `extra_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. +Credential fields contain GitHub Variable/Secret names, never credential values. `apps`, `app_refs`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. The app list is rendered into the encrypted asset as a comma-separated `APPS` value. `app_refs` must list at least one app bundle — flightdeck's own `apps/` catalog is just another entry, not implicit. `load-yaml-matrix` reads every file in `vaults/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. @@ -526,7 +528,7 @@ Secrets take precedence over Variables when both contain the same source key. Ev ### `build-bundle` -Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. Callers choose the archive name and contents, so the same action publishes the core `flightdeck.zip` bundle or a consumer repository's `flightdeck-extra.zip` bundle. +Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. Callers choose the archive name and contents, so the same action publishes flightdeck's own machinery bundle (`flightdeck.zip`), its own `apps/` catalog (`flightdeck-apps.zip`), or a consumer repository's own app bundle. ```yaml steps: @@ -536,12 +538,12 @@ steps: - uses: rubykatzen/flightdeck/.github/actions/build-bundle@v1.2.3 with: paths: apps - bundle-name: flightdeck-extra.zip + bundle-name: flightdeck-apps.zip release-tag: v1.2.3 token: ${{ secrets.GITHUB_TOKEN }} ``` -Requires `contents: write` permission on the calling job. `flightdeck-extra.zip` is the default asset name expected by `flightdeck_extra_refs`; use an explicit `:asset-name` suffix in the ref when publishing a different filename. +Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` is the default asset name a `flightdeck_app_refs` entry resolves to when it doesn't specify an explicit `:asset-name` suffix; use that suffix when publishing under a different filename. --- @@ -561,7 +563,7 @@ jobs: hosts: '["deploy@100.64.0.1", "deploy@100.64.0.2"]' # required JSON array app-ref: rubykatzen/flightdeck@latest # required full release ref env-ref: "${{ github.repository }}@latest:.sops.env" # required - # extra-refs: '["owner/repo@latest"]' # optional JSON array, default: [] + app-refs: '["rubykatzen/flightdeck@latest"]' # required non-empty JSON array # path: ~/flightdeck # optional, default shown # keep-releases: 5 # optional, default shown # sops-age-key-file: /home/deploy/.config/sops/age/keys.txt # optional, default: ~/.config/sops/age/keys.txt for `user` diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 1a96bd8..279bebf 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -11,11 +11,11 @@ that: - flightdeck_app_ref is defined and flightdeck_app_ref | length > 0 - flightdeck_env_ref is defined and flightdeck_env_ref | length > 0 - - flightdeck_extra_refs is defined + - flightdeck_app_refs is defined and flightdeck_app_refs | length > 0 - flightdeck_path is defined and flightdeck_path | length > 0 - flightdeck_keep_releases is defined - flightdeck_sops_age_key_file is defined and flightdeck_sops_age_key_file | length > 0 - fail_msg: "Set flightdeck_app_ref, flightdeck_env_ref, flightdeck_extra_refs, flightdeck_path, flightdeck_keep_releases, and flightdeck_sops_age_key_file" + fail_msg: "Set flightdeck_app_ref, flightdeck_env_ref, flightdeck_app_refs, flightdeck_path, flightdeck_keep_releases, and flightdeck_sops_age_key_file" - name: Resolve Flightdeck user paths set_fact: flightdeck_user_home: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}" @@ -103,16 +103,20 @@ src: "{{ flightdeck_app_pull.path }}/flightdeck.zip" dest: "{{ flightdeck_release_path }}" remote_src: true - - name: Create extra packages pull directory + - name: Ensure release apps directory + file: + path: "{{ flightdeck_release_path }}/apps" + state: directory + mode: "0755" + - name: Create app packages pull directory tempfile: state: directory - suffix: flightdeck-extra-packages - register: flightdeck_extra_pull - when: flightdeck_extra_refs | length > 0 - - name: Pull and merge extra app packages + suffix: flightdeck-app-packages + register: flightdeck_app_packages_pull + - name: Pull and merge app packages shell: | set -euo pipefail - extra_root={{ flightdeck_extra_pull.path | quote }} + packages_root={{ flightdeck_app_packages_pull.path | quote }} release_apps={{ (flightdeck_release_path + '/apps') | quote }} download_release_ref() { ref="$1" @@ -145,20 +149,20 @@ if [ -n "${FLIGHTDECK_GITHUB_TOKEN:-}" ]; then export GH_TOKEN="$FLIGHTDECK_GITHUB_TOKEN" fi - {% for ref in flightdeck_extra_refs %} - package_dir="$extra_root/{{ loop.index }}" + {% for ref in flightdeck_app_refs %} + package_dir="$packages_root/{{ loop.index }}" mkdir -p "$package_dir/pull" "$package_dir/extract" - bundle="$(download_release_ref {{ ref | quote }} "$package_dir/pull" flightdeck-extra.zip)" + bundle="$(download_release_ref {{ ref | quote }} "$package_dir/pull" flightdeck-apps.zip)" unzip "$bundle" -d "$package_dir/extract" if [ ! -d "$package_dir/extract/apps" ]; then - echo "Extra package {{ ref }} does not contain apps/" >&2 + echo "Package {{ ref }} does not contain apps/" >&2 exit 1 fi for app_path in "$package_dir/extract/apps"/*; do [ -d "$app_path" ] || continue app="$(basename "$app_path")" if [ -e "$release_apps/$app" ]; then - echo "Extra app conflicts with an existing app: $app" >&2 + echo "App conflicts with an existing app: $app" >&2 exit 1 fi cp -a "$app_path" "$release_apps/" @@ -168,7 +172,6 @@ executable: /bin/bash environment: FLIGHTDECK_GITHUB_TOKEN: "{{ flightdeck_github_token }}" - when: flightdeck_extra_refs | length > 0 - name: Create env package pull directory tempfile: state: directory @@ -281,8 +284,8 @@ path: "{{ flightdeck_env_pull.path }}" state: absent when: flightdeck_env_pull is defined and flightdeck_env_pull.path is defined - - name: Remove extra packages pull directory + - name: Remove app packages pull directory file: - path: "{{ flightdeck_extra_pull.path }}" + path: "{{ flightdeck_app_packages_pull.path }}" state: absent - when: flightdeck_extra_pull is defined and flightdeck_extra_pull.path is defined + when: flightdeck_app_packages_pull is defined and flightdeck_app_packages_pull.path is defined diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 5f91afd..387f034 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,6 +1,7 @@ flightdeck_ref: rubykatzen/flightdeck@latest env_ref: rubykatzen/flightdeck@latest:hawkeye.sops.env -extra_refs: [] +app_refs: + - rubykatzen/flightdeck@latest hosts: - rubykatzen-com@100.75.50.2 credentials: From 5976f34a7144a7c1d33911a9af69a01f777f613a Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Thu, 20 Aug 2026 00:13:12 +0200 Subject: [PATCH 34/35] refactor: split build-bundle, push paths/bundle-name into overridable defaults build-bundle's paths and bundle-name now default to Flightdeck's own machinery bundle instead of being required on every call, so the canonical file list lives in one place (the action) instead of inline in release.yml. build-apps-bundle is a new, thin defaults wrapper around build-bundle (paths: apps, bundle-name: flightdeck-apps.zip) used both for flightdeck's own apps/ catalog and by any consumer repo publishing its own app bundle the same way. --- .github/actions/build-apps-bundle/action.yml | 31 +++++++++++++++++++ .github/actions/build-bundle/action.yml | 18 +++++++++-- .github/workflows/release.yml | 17 +---------- README.md | 32 ++++++++++++++++---- 4 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 .github/actions/build-apps-bundle/action.yml diff --git a/.github/actions/build-apps-bundle/action.yml b/.github/actions/build-apps-bundle/action.yml new file mode 100644 index 0000000..52d522b --- /dev/null +++ b/.github/actions/build-apps-bundle/action.yml @@ -0,0 +1,31 @@ +name: Build and upload apps bundle +description: Build an apps/ catalog bundle and upload it as a GitHub Release asset. Thin defaults wrapper around build-bundle. +inputs: + paths: + description: Newline-separated paths to include in the bundle. + required: false + default: apps + bundle-name: + description: Bundle archive filename. + required: false + default: flightdeck-apps.zip + release-tag: + description: Release tag to upload the bundle asset to. + required: true + token: + description: Token with permission to upload release assets. + required: true +outputs: + bundle-path: + description: Path to the built bundle archive. + value: ${{ steps.build.outputs.bundle-path }} +runs: + using: composite + steps: + - uses: ./.github/actions/build-bundle + id: build + with: + paths: ${{ inputs.paths }} + bundle-name: ${{ inputs.bundle-name }} + release-tag: ${{ inputs.release-tag }} + token: ${{ inputs.token }} diff --git a/.github/actions/build-bundle/action.yml b/.github/actions/build-bundle/action.yml index c32e890..54c192b 100644 --- a/.github/actions/build-bundle/action.yml +++ b/.github/actions/build-bundle/action.yml @@ -1,13 +1,25 @@ name: Build and upload bundle -description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. +description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. Defaults to Flightdeck's own machinery bundle. inputs: paths: description: Newline-separated paths to include in the bundle. - required: true + required: false + default: | + ansible.cfg + .env.example + backup.sh + deploy.sh + down.sh + generate-env.sh + lib.sh + logs.sh + restart.sh + up.sh + README.md bundle-name: description: Bundle archive filename. required: false - default: bundle.zip + default: flightdeck.zip release-tag: description: Release tag to upload the bundle asset to. required: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aad278b..3676911 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,21 +27,8 @@ jobs: ref: ${{ needs.release.outputs.tag_name }} - uses: $/.github/actions/build-bundle with: - bundle-name: flightdeck.zip release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} - paths: | - ansible.cfg - .env.example - backup.sh - deploy.sh - down.sh - generate-env.sh - lib.sh - logs.sh - restart.sh - up.sh - README.md upload-apps: needs: release if: needs.release.outputs.release_created == 'true' @@ -50,12 +37,10 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/build-bundle + - uses: $/.github/actions/build-apps-bundle with: - bundle-name: flightdeck-apps.zip release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} - paths: apps encrypt-vaults: needs: release if: needs.release.outputs.release_created == 'true' diff --git a/README.md b/README.md index cfbde27..dc97e78 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,8 @@ flightdeck/ │ └── deploy.yml # Deploy published bundle and encrypted env ├── .github/ │ ├── actions/ -│ │ ├── build-bundle/ # Build and upload a release bundle +│ │ ├── build-bundle/ # Build and upload the machinery bundle +│ │ ├── build-apps-bundle/ # Build and upload an apps/ catalog bundle │ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release │ │ └── load-yaml-matrix/ # Read a directory of YAML manifests into a workflow matrix │ └── workflows/ @@ -442,7 +443,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d ## ⚙️ GitHub Actions -This repository provides three composite actions under `.github/actions/` (`build-bundle`, `encrypt-env`, and `load-yaml-matrix`) and one reusable workflow, `deploy-shared.yml`. +This repository provides four composite actions under `.github/actions/` (`build-bundle`, `build-apps-bundle`, `encrypt-env`, and `load-yaml-matrix`) and one reusable workflow, `deploy-shared.yml`. --- @@ -528,7 +529,7 @@ Secrets take precedence over Variables when both contain the same source key. Ev ### `build-bundle` -Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. Callers choose the archive name and contents, so the same action publishes flightdeck's own machinery bundle (`flightdeck.zip`), its own `apps/` catalog (`flightdeck-apps.zip`), or a consumer repository's own app bundle. +Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. `paths` and `bundle-name` default to Flightdeck's own machinery bundle (everything except `apps/`, uploaded as `flightdeck.zip`) but are fully overridable. ```yaml steps: @@ -537,13 +538,32 @@ steps: ref: v1.2.3 - uses: rubykatzen/flightdeck/.github/actions/build-bundle@v1.2.3 with: - paths: apps - bundle-name: flightdeck-apps.zip release-tag: v1.2.3 token: ${{ secrets.GITHUB_TOKEN }} + # paths: ... # optional, defaults to the machinery file list + # bundle-name: ... # optional, defaults to flightdeck.zip ``` -Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` is the default asset name a `flightdeck_app_refs` entry resolves to when it doesn't specify an explicit `:asset-name` suffix; use that suffix when publishing under a different filename. +Requires `contents: write` permission on the calling job. + +--- + +### `build-apps-bundle` + +A thin defaults wrapper around `build-bundle`: `paths` defaults to `apps`, `bundle-name` defaults to `flightdeck-apps.zip`. The same action publishes flightdeck's own `apps/` catalog and any consumer repository's own app bundle. + +```yaml +steps: + - uses: actions/checkout@v7 + with: + ref: v1.2.3 + - uses: rubykatzen/flightdeck/.github/actions/build-apps-bundle@v1.2.3 + with: + release-tag: v1.2.3 + token: ${{ secrets.GITHUB_TOKEN }} +``` + +Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` is the default asset name a `flightdeck_app_refs` entry resolves to when it doesn't specify an explicit `:asset-name` suffix; override `bundle-name` and use that suffix when publishing under a different filename. --- From e43cfd7439a7e691868858640be5c8a0539ee705 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Thu, 20 Aug 2026 02:33:19 +0200 Subject: [PATCH 35/35] docs: mark path/sops_age_key_file as optional in the targets example --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dc97e78..d6fd0e7 100644 --- a/README.md +++ b/README.md @@ -475,8 +475,8 @@ app_refs: hosts: - deploy@100.64.0.1 - deploy@100.64.0.2 -path: ~/flightdeck -sops_age_key_file: ~/.config/sops/age/keys.txt +path: ~/flightdeck # optional, default shown +sops_age_key_file: ~/.config/sops/age/keys.txt # optional, default shown credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID