diff --git a/.env.example b/.env.example deleted file mode 100644 index acadc2f..0000000 --- a/.env.example +++ /dev/null @@ -1,58 +0,0 @@ -# System -APPS=traefik -APPS_DOMAIN=... -APPS_HOST=... -APPS_HOST_NAME=... -APPS_INTERNAL_IP=... -APPS_ADMIN_MAIL=... -APPS_ADMIN_DEFAULT_PASS=... -APPS_CERTIFICATE_RESOLVER=... -APPS_CLOUDFLARE_DNS_API_TOKEN=... -APPS_TIMEZONE=... -APPS_UID=... -APPS_GID=... -# Database and secrets -APPS_DATABASE_PASSWORD=... -APPS_KEY_HEX_16=... -APPS_KEY_HEX_32=... -APPS_KEY_HEX_64=... -APPS_KEY_BASE64_32=... -APPS_HTPASSWD=... -# OAuth -APPS_GOOGLE_OAUTH_CLIENT_ID=... -APPS_GOOGLE_CLIENT_SECRET=... -# Notifications -APPS_TELEGRAM_TOKEN=... -APPS_TELEGRAM_CHAT=... -APPS_SHOUTRRR_TELEGRAM_NOTIFICATION_URL=... -# SMTP -APPS_SMTP_HOSTNAME=... -APPS_SMTP_PORT=... -APPS_SMTP_USERNAME=... -APPS_SMTP_PASSWORD=... -APPS_SMTP_FROM=... -# S3 -APPS_S3_HOST=... -APPS_S3_REGION=... -APPS_S3_ACCESS_KEY_ID=... -APPS_S3_SECRET_ACCESS_KEY=... -# Apps -TRAEFIK_HTTP_PORT=... -TRAEFIK_HTTPS_PORT=... -BESZEL_AGENT_PUBLIC_KEY=... -BESZEL_DISK_1_DEVICE=... -BESZEL_DISK_2_DEVICE=... -RSSHUB_ACCESS_KEY=... -RSSHUB_TWITTER_AUTH_TOKEN=... -RSSHUB_YOUTUBE_KEY=... -ICLOUDPD_USERNAME=... -ICLOUDPD_PASSWORD=... -ICLOUDPD_ALBUM=... -ICLOUDPD_DATA_PATH=... -CODECOV_LICENSE=... -CODECOV_ADMIN_GITHUB_USERNAME=... -CODECOV_GITHUB_CLIENT_ID=... -CODECOV_GITHUB_CLIENT_SECRET=... -CODECOV_GITHUB_WEBHOOK_SECRET=... -CODECOV_GITHUB_APP_ID=... -CODECOV_S3_BUCKET=... diff --git a/.github/actions/build-bundle/action.yml b/.github/actions/build-bundle/action.yml index 54c192b..4c15b20 100644 --- a/.github/actions/build-bundle/action.yml +++ b/.github/actions/build-bundle/action.yml @@ -1,25 +1,12 @@ name: Build and upload bundle -description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. Defaults to Flightdeck's own machinery bundle. +description: Build a zip bundle from specified paths and upload it as a GitHub Release asset. inputs: paths: description: Newline-separated paths to include in the bundle. - required: false - default: | - ansible.cfg - .env.example - backup.sh - deploy.sh - down.sh - generate-env.sh - lib.sh - logs.sh - restart.sh - up.sh - README.md + required: true bundle-name: description: Bundle archive filename. - required: false - default: flightdeck.zip + required: true release-tag: description: Release tag to upload the bundle asset to. required: true diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml index 54bf3fc..705a52c 100644 --- a/.github/workflows/deploy-shared.yml +++ b/.github/workflows/deploy-shared.yml @@ -6,20 +6,12 @@ on: description: JSON array of user@host SSH destinations to deploy to. type: string required: true - app-ref: - description: Full release ref of the Flightdeck bundle to deploy, in owner/repo@tag format. - type: string - required: true - env-refs: - description: JSON array of release refs for encrypted env packages to decrypt and merge, in owner/repo@tag[:asset] format. - type: string - required: true app-refs: description: JSON array of release refs for app bundles to merge into the release, in owner/repo@tag[:asset] format. type: string required: true apps: - description: JSON array of app names to run on this target, rendered into the deployed env as APPS. + description: JSON object mapping each app name to run on this target to its own env_refs list, e.g. {"traefik":{"env_refs":["owner/repo@latest:traefik.sops.env"]}}. type: string required: true path: @@ -30,10 +22,6 @@ on: description: Number of past releases to keep on the target host. type: number default: 5 - sops-age-key-file: - description: Path to the server-local SOPS age key file, relative to each SSH user's home when it starts with ~. - type: string - default: "~/.config/sops/age/keys.txt" tailscale-oauth-client-id: description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). type: string @@ -46,6 +34,9 @@ on: ssh-private-key: description: SSH private key used to connect to the hosts. required: true + sops-age-key: + description: Private SOPS age key used to decrypt this target's vault-sourced env, on the runner. + required: true tailscale-oauth-secret: description: Tailscale OAuth client secret used to join the tailnet. Required only when tailscale-oauth-client-id is set. required: false @@ -57,9 +48,18 @@ jobs: with: repository: ${{ job.workflow_repository }} ref: ${{ job.workflow_sha }} - - name: Install ansible-core + - name: Install deploy dependencies + shell: bash + run: pip install --user --break-system-packages -r deploy/requirements.txt + - name: Install sops shell: bash - run: pip install --user --break-system-packages ansible-core + env: + SOPS_VERSION: "3.13.1" + run: | + curl --fail --location --silent --show-error \ + "https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64" \ + --output /usr/local/bin/sops + chmod +x /usr/local/bin/sops - uses: tailscale/github-action@v4 if: inputs.tailscale-oauth-client-id != '' with: @@ -73,45 +73,22 @@ jobs: echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" ssh-add - <<< "${{ secrets.ssh-private-key }}" - - name: Build extra-vars - id: vars + - name: Run deploy shell: bash env: - APP_REF: ${{ inputs.app-ref }} - ENV_REFS: ${{ inputs.env-refs }} + HOSTS: ${{ inputs.hosts }} APP_REFS: ${{ inputs.app-refs }} APPS: ${{ inputs.apps }} - HOSTS: ${{ inputs.hosts }} DEPLOY_PATH: ${{ inputs.path }} KEEP_RELEASES: ${{ inputs.keep-releases }} - SOPS_KEY_FILE: ${{ inputs.sops-age-key-file }} + SOPS_AGE_KEY: ${{ secrets.sops-age-key }} run: | - hosts_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and test("^[a-z_][a-z0-9_-]*@[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$")) then . else error("hosts must be a non-empty user@host string array") end' <<< "$HOSTS")" - env_refs_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("env-refs must be a non-empty string array") end' <<< "$ENV_REFS")" - app_refs_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("app-refs must be a non-empty string array") end' <<< "$APP_REFS")" - apps_json="$(jq -ce 'if type == "array" and length > 0 and all(.[]; type == "string" and length > 0) then . else error("apps must be a non-empty string array") end' <<< "$APPS")" - jq -ce ' - reduce .[] as $destination ({all: {hosts: {}}}; - ($destination | capture("^(?[^@]+)@(?.+)$")) as $ssh | - .all.hosts[$ssh.host] = {ansible_user: $ssh.user} - ) - ' <<< "$hosts_json" > "$RUNNER_TEMP/flightdeck-inventory.json" - json="$(jq -n \ - --arg app_ref "$APP_REF" \ - --argjson env_refs "$env_refs_json" \ - --argjson app_refs "$app_refs_json" \ - --argjson apps "$apps_json" \ + jq -n \ + --argjson hosts "$HOSTS" \ + --argjson app_refs "$APP_REFS" \ + --argjson apps "$APPS" \ --arg path "$DEPLOY_PATH" \ --argjson keep_releases "$KEEP_RELEASES" \ - --arg sops_key_file "$SOPS_KEY_FILE" \ - '{flightdeck_app_ref: $app_ref, flightdeck_env_refs: $env_refs, flightdeck_app_refs: $app_refs, flightdeck_apps: $apps, flightdeck_path: $path, flightdeck_keep_releases: $keep_releases, flightdeck_sops_age_key_file: $sops_key_file}')" - echo "json=$json" >> "$GITHUB_OUTPUT" - echo "inventory=$RUNNER_TEMP/flightdeck-inventory.json" >> "$GITHUB_OUTPUT" - - name: Run playbook - shell: bash - env: - ANSIBLE_HOST_KEY_CHECKING: "false" - run: | - ansible-playbook ansible/deploy.yml \ - -i "${{ steps.vars.outputs.inventory }}" \ - -e "${{ steps.vars.outputs.json }}" + --arg sops_age_key "$SOPS_AGE_KEY" \ + '{hosts: $hosts, app_refs: $app_refs, apps: $apps, path: $path, keep_releases: $keep_releases, sops_age_key: $sops_age_key}' \ + | python3 deploy/deploy.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8dbb31f..a605a96 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -27,14 +27,12 @@ jobs: uses: $/.github/workflows/deploy-shared.yml with: hosts: ${{ toJson(matrix.hosts) }} - app-ref: ${{ matrix.flightdeck_ref }} - env-refs: ${{ toJson(matrix.env_refs) }} app-refs: ${{ toJson(matrix.app_refs) }} apps: ${{ toJson(matrix.apps) }} path: ${{ matrix.path || '~/flightdeck' }} keep-releases: ${{ matrix.keep_releases || 5 }} - sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} secrets: ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} + sops-age-key: ${{ secrets[matrix.credentials.secrets.sops_age_key] }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0962f4d..9bf2817 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,18 +17,6 @@ jobs: id: release with: token: ${{ secrets.RELEASE_TOKEN }} - upload: - needs: release - if: needs.release.outputs.release_created == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/build-bundle - with: - release-tag: ${{ needs.release.outputs.tag_name }} - token: ${{ secrets.GITHUB_TOKEN }} upload-apps: needs: release if: needs.release.outputs.release_created == 'true' @@ -90,21 +78,19 @@ jobs: with: directory: targets deploy: - needs: [upload, upload-apps, encrypt, deploy-targets] + needs: [upload-apps, encrypt, deploy-targets] if: needs.deploy-targets.outputs.count != '0' strategy: matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} uses: $/.github/workflows/deploy-shared.yml with: hosts: ${{ toJson(matrix.hosts) }} - app-ref: ${{ matrix.flightdeck_ref }} - env-refs: ${{ toJson(matrix.env_refs) }} app-refs: ${{ toJson(matrix.app_refs) }} apps: ${{ toJson(matrix.apps) }} path: ${{ matrix.path || '~/flightdeck' }} keep-releases: ${{ matrix.keep_releases || 5 }} - sops-age-key-file: ${{ matrix.sops_age_key_file || '~/.config/sops/age/keys.txt' }} tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} secrets: ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} + sops-age-key: ${{ secrets[matrix.credentials.secrets.sops_age_key] }} diff --git a/.gitignore b/.gitignore index 072c599..26c0061 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ __pycache__ /apps-data/* -/.env /apps.env -/backups/ /apps/*/.env .claude/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57c03d5..faf426c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,5 +5,4 @@ repos: - id: yamllint - id: pymarkdown - id: ruff - - id: shellcheck - id: actionlint diff --git a/AGENTS.md b/AGENTS.md index ff0699a..4d41f45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,16 +50,15 @@ A repository-specific exception must be declared through the `skip` input of ## Repository Overview -This is a Docker-based application management system (flightdeck) that orchestrates core self-hosted services using docker-compose. The architecture uses Traefik as a reverse proxy with automatic SSL certificate management, and can merge optional extra application catalogs during Ansible deployment. +This is a Docker-based application management system (flightdeck) that orchestrates core self-hosted services using docker-compose. The architecture uses Traefik as a reverse proxy with automatic SSL certificate management, and can merge optional extra application catalogs during deploy. There is no manual administration flow at all - no server-side console access, no local quick-start. Every deploy goes through `targets/`/`vaults/` manifests and GitHub Actions (`deploy/deploy.py`); see the "Environment Variable System" and "CI/CD" sections below. ## Core Architecture ### Directory Structure - `apps/` - Contains core docker-compose configurations and shared compose templates -- `apps-data/` - Persistent data storage for all running applications -- `.env` - All environment variables for this server: shared `APPS_*`, per-app overrides, and the comma-separated `APPS` list. Deployed via Ansible from encrypted secrets release assets. -- Shell scripts at root for orchestration +- `apps-data/` - Persistent data storage on the target host (not in this repo): each app's data plus its rendered config +- `deploy/` - The push-based deploy entrypoint and its supporting modules (ref resolution, collision detection, decryption, config rendering), run on the GitHub Actions runner ### Docker Compose Architecture @@ -71,29 +70,30 @@ The repository uses a modular docker-compose structure with reusable components: - `x-restart`: Restart policy (unless-stopped) - Pre-defined service profiles: `main`, `main-http`, `api`, `host`, `side` -2. **Shared Infrastructure** (`apps/networks.yml`, `apps/postgres.yml`, `apps/redis.yml`): +2. **Shared Infrastructure** (`apps/networks.yml` plus versioned database/cache/service templates): - `networks.yml`: Defines `internal`, `databases`, `mcp`, and `traefik` networks - - `postgres.yml`: PostgreSQL 17 service template - - `redis.yml`: Redis 7 service template - - Apps include these via `include:` directive to get database/cache services + - `postgres-17.yml`, `postgres-18.yml`: PostgreSQL service templates + - `paradedb-17.yml`, `pgvector-17.yml`: Postgres-compatible variants (full-text search, vector search) + - `redis-7.yml`, `redis-8.yml`: Redis service templates + - `mysql-8.yml`: MySQL service template + - `mongodb-8.yml`: MongoDB service template + - `clickhouse-25.4.yml`, `clickhouse-26.5.yml`: ClickHouse service templates + - `timescale-17.yml`: TimescaleDB service template + - `gotenberg-8.yml`: Gotenberg (document conversion) service template + - Templates are versioned by filename (e.g. `postgres-17.yml` vs `postgres-18.yml`) so an app picks its version explicitly via which file it includes, not a shared default + - Apps include these via `include:` directive to get database/cache/service dependencies 3. **App Structure Pattern**: Each app in `apps/` has: - - `.env` file generated by `generate-env.sh` containing `APP_NAME`, shared `APPS_*`, and only that app's prefixed variables (gitignored) - `docker-compose.yml` extending common services - Optional `config/` with template files (`.template.yml`) + - A `.env` on the target host only, decrypted and placed there by `deploy/deploy.py` (never checked into this repo, never present until a real deploy runs) ### Environment Variable System -Environment variables come from a single `.env` file deployed by Ansible from an encrypted secrets release asset. It contains shared `APPS_*` variables, per-app variables, and the comma-separated `APPS` list. +There is no root `.env` anywhere - not on a target host, not locally. Each app's env comes entirely from that app's own vault(s), declared in `targets/{target}.yml`'s `apps..env_refs` (see README's "Vaults And Targets"). `deploy/deploy.py` runs on the GitHub Actions runner: it downloads each app's still-encrypted vault assets, checks their key names for collisions from the ciphertext directly (no decryption needed for that check), decrypts them with the target's private SOPS age key, concatenates the plaintext, and writes it straight into that app's `.env` in the release tree before pushing. `deploy/render.py`'s `render_template` then does the same substitution `envsubst` would, also on the runner, for that app's `config/*.template.*` files, using the just-decrypted values. -Before starting each app, `up.sh` runs `generate-env.sh`, which calls `generate_env` from `lib.sh`. The generated `apps/{app}/.env` contains: - -- `APP_NAME` from the app folder name -- all shared variables beginning with `APPS_` -- only variables beginning with the normalized app prefix, e.g. `TWOFAUTH_*` for `twofauth` or `BESZEL_AGENT_*` for `beszel-agent` - -Variables for one app must not be visible to other apps. This allows running docker compose directly from the app folder without any `--env-file` flags while keeping app secrets scoped. +A vault declares the exact final variable name an app receives directly (e.g. `HTTP_PORT`, not `TRAEFIK_HTTP_PORT`) - there is no automatic prefix-stripping or filtering step anywhere. Variables for one app are never visible to another app, since each app's `.env` is built from that app's own vault(s) only. This allows running docker compose directly from the app folder without any `--env-file` flags while keeping app secrets scoped. ### Per-app and per-server overrides @@ -117,9 +117,8 @@ Naming convention: The compose file is the source of truth for which overrides are allowed. Not every variable needs an app-specific override — only declare one when you actually want to allow it. -Key variables in `.env`: +Common `APPS_*` variables a target's vaults map into one or more apps' `.env`: -- `APPS` - Comma-separated app names to deploy on this server, e.g. `traefik,gatus,semaphore` - `APPS_DOMAIN` - Base domain for all services - `APPS_CERTIFICATE_RESOLVER` - SSL resolver (Cloudflare DNS or HTTP challenge) - `APPS_DATABASE_PASSWORD` - Shared database password @@ -139,73 +138,11 @@ traefik.http.services.${APP_NAME}.loadbalancer.server.port=8080 Apps are accessible at `{app-name}.{APPS_DOMAIN}` with automatic SSL. -## Common Commands - -### Starting Applications - -```bash -# Start all apps defined in APPS in .env -./up.sh - -# Start specific app(s) -./up.sh traefik gatus rybbit -``` - -The `up.sh` script: - -- Generates `apps/{app}/.env` with shared `APPS_*` variables and only that app's own prefixed variables -- Sources the merged env for template processing -- Processes config templates using envsubst (files matching `*.template.*`) -- Creates apps-data directories -- Runs docker compose up for each app - -### Stopping Applications - -```bash -# Stop all apps -./down.sh - -# Stop specific app(s) -./down.sh traefik gatus -``` - -### Restarting Applications - -```bash -# Restart all apps (down + up) -./restart.sh - -# Restart specific app(s) -./restart.sh rybbit -``` - -### Viewing Logs - -```bash -# View logs for a specific app (requires single app name) -./logs.sh gatus -``` - -### Backing Up Apps +## Operations -```bash -# Back up all apps-data directories from a remote server to local backups/ -./backup.sh root@server1.example.com -./backup.sh root@server2.example.com -``` - -The `backup.sh` script stops each active app, zips its `apps-data/` directory, downloads it locally, then restarts the app. Inactive apps (not in APPS) are archived without stopping. +There are no wrapper scripts and nothing runs them - starting, stopping, and restarting apps all happen by deploying (`deploy/deploy.py`, see "CI/CD" below). `apps-data/traefik/acme.json` and the external Docker networks `traefik`, `databases`, and `mcp` are created idempotently by `deploy/deploy.py` on every deploy (derived from `apps/networks.yml`'s `external: true` entries), not by a separate first-run step. -### Updating Applications - -```bash -# up.sh pulls latest images automatically before starting -./restart.sh -``` - -### Direct Docker Compose Commands - -After `./up.sh ` has generated `apps/{app}/.env`, docker compose works directly from the app folder without any flags: +Debugging an already-deployed app means SSHing into the target host directly and using Docker Compose itself - no wrapper needed, since each app's directory is already a complete, ready-to-run Compose project (real `.env` sitting next to the compose file): ```bash cd apps/gatus && docker compose logs -f @@ -214,15 +151,7 @@ cd apps/gatus && docker compose exec gatus sh cd apps/traefik && docker compose down ``` -## Initial Setup - -The `up.sh` script performs first-run setup automatically before loading environment variables: - -```bash -./up.sh -``` - -It creates `.env` from `.env.example`, `apps-data/traefik/acme.json`, and the external Docker networks `traefik`, `databases`, and `mcp` when missing. +Backups are a separate, not-yet-decided piece of tooling (the old `backup.sh` assumed a human running it from their own machine over SSH, which no longer fits). ## Adding New Applications @@ -230,11 +159,11 @@ It creates `.env` from `.env.example`, `apps-data/traefik/acme.json`, and the ex 2. Create `docker-compose.yml`: - Include `../networks.yml` for network definitions - Extend `../common.yml` service definitions (usually `main`) - - Include `../postgres.yml` and/or `../redis.yml`, `../mongo.yml` if needed + - Include a versioned database/service template if needed, e.g. `../postgres-18.yml`, `../redis-8.yml`, `../mongodb-8.yml` (see "Shared Infrastructure" above for the full list) - Reference data path: `../../apps-data/${APP_NAME}/` - Set the service port explicitly with `expose` and `traefik.http.services.${APP_NAME}.loadbalancer.server.port` -3. Add app name to the comma-separated `APPS` list in `.env` -4. If app needs configuration templates, create `config/{name}.template.yml` (envsubst will process) +3. Wire it into a target's `apps` mapping and give it a vault declaring the env it needs (see README's "Vaults And Targets") +4. If app needs configuration templates, create `config/{name}.template.yml` (`deploy/render.py` processes these on the runner during deploy, the same substitution `envsubst` would do) Example minimal app structure: @@ -276,9 +205,9 @@ To maintain consistency across all applications, follow these strict field order # 1. INCLUDE DIRECTIVES (always first) include: - ../networks.yml # Always first - - ../postgres.yml # If PostgreSQL needed - - ../redis.yml # If Redis needed - - ../mongo.yml # If MongoDB needed + - ../postgres-18.yml # If PostgreSQL needed (pick a version, see "Shared Infrastructure") + - ../redis-8.yml # If Redis needed (pick a version) + - ../mongodb-8.yml # If MongoDB needed # 2. X-IMAGE (if multiple services use the same image) x-image: &image @@ -350,7 +279,7 @@ services: ### Key ordering principles: -1. **Include order**: networks.yml → postgres.yml → redis.yml → mongo.yml +1. **Include order**: networks.yml → database/cache templates (postgres/redis/mongodb/etc., pick a version) → others 2. **X-fields order**: x-image → x-environment → x-volumes (only if needed) 3. **X-image for shared images** - if multiple services use the same image, use `x-image: &image` 4. **X-volumes for shared volumes** - if 2+ volumes repeat across services, extract them to `x-volumes: &volumes` and merge with unique ones @@ -380,7 +309,7 @@ Example: ```yaml include: - ../networks.yml - - ../postgres.yml + - ../postgres-18.yml x-environment: &environment DATABASE_URL: postgresql://postgres:${APPS_DATABASE_PASSWORD}@postgres:5432/${APP_NAME} ENABLE_FEATURE: true @@ -400,34 +329,33 @@ services: ## CI/CD -GitHub Actions workflow (`.github/workflows/release-please.yml`) manages releases via [Release Please](https://github.com/googleapis/release-please): +GitHub Actions workflow (`.github/workflows/release.yml`) manages releases via [Release Please](https://github.com/googleapis/release-please): 1. On every push to `main`, Release Please opens/updates a `chore(main): release X.Y.Z` PR with the computed version and generated `CHANGELOG.md` entry 2. Merging that PR tags the release and publishes a GitHub Release -3. A second and third job then build and upload two release assets: `flightdeck.zip` from helper scripts, examples, and README (verifying runtime state such as `.env`, `apps-data`, `backups`, and generated `apps/*/.env` is excluded), and `flightdeck-apps.zip` from the `apps/` catalog alone. Deploy refs may use `@latest` as a playbook-side alias for GitHub's latest release API; no mutable `latest` release/tag is created. +3. A job then builds and uploads `flightdeck-apps.zip` from the `apps/` catalog. Deploy refs may use `@latest` as an alias resolved through GitHub's latest release API (`deploy/resolve.py`); no mutable `latest` release/tag is created. -Deployment helpers live in this repository: +Deployment helpers live in this repository, entirely under `deploy/`, run only on the GitHub Actions runner - the target host never runs any of this: -- `ansible/deploy.yml` pulls `flightdeck_app_ref` (the machinery bundle), merges every ref in `flightdeck_app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit), decrypts and merges every ref in `flightdeck_env_refs` (at least one required) into the server's `.env`, switches a timestamped release, and runs `./deploy.sh` +- `deploy/deploy.py` is the deploy entrypoint. It resolves and downloads every ref in `app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit) and merges them into a release tree locally; for each app in the target's `apps` mapping, downloads its `env_refs` (still encrypted), decrypts them with the target's private SOPS age key, writes the plaintext into that app's `.env` in the release tree, and renders that app's `config/*.template.*` files with the decrypted values. It then opens an SSH connection per host, pushes the finished release (real `.env`, already-rendered config), bootstraps networks/directories idempotently, switches a timestamped release, and runs `docker compose pull && docker compose up -d` per app directly (no wrapper script on the host at all). +- `deploy/resolve.py`, `deploy/collisions.py`, `deploy/vault.py`, and `deploy/render.py` hold, respectively, the ref-resolution, ciphertext collision-detection, decryption, and template-rendering logic - each with real `unittest` coverage in `deploy/tests/`. - `.github/actions/encrypt-env/` is a local composite action for rendering `vaults/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset — vault manifests hold only env/secrets, not app selection -- `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `ansible/deploy.yml` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository - -App bundles listed in `flightdeck_app_refs` are release assets referenced as short refs like `/@latest` or `/@v1.2.3`, resolving to a default asset name of `flightdeck-apps.zip` unless the ref specifies an explicit `:asset-name` suffix. `@latest` is resolved by the deploy playbook through GitHub's latest release API. Every bundle must contain an `apps/` directory; app names may not conflict across bundles. +- `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `deploy/deploy.py` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository -Env packages listed in `flightdeck_env_refs` are release refs the same shape as app bundles, defaulting to a `$tag.sops.env`-named asset. The playbook decrypts each with the server-local SOPS age key, then merges them alongside a synthesized `APPS` line (built from `flightdeck_apps`, the target's own desired app set) — failing loud on any key collision across sources, `APPS` included. `apps` moved off the vault schema onto the target for exactly this reason: multiple vaults can be merged without having to reconcile per-vault `apps` lists. +App bundles listed in `app_refs` are release assets referenced as short refs like `/@latest` or `/@v1.2.3`, resolving to a default asset name of `flightdeck-apps.zip` unless the ref specifies an explicit `:asset-name` suffix. `@latest` is resolved through GitHub's latest release API. Every bundle must contain an `apps/` directory; both per-app directories and shared top-level files (`common.yml`, `networks.yml`, etc.) merge the same way — copy if new, fail loud on any name conflict across bundles. -Private release assets are supported by passing `FLIGHTDECK_GITHUB_TOKEN` as a secret environment variable to `ansible/deploy.yml`. Store it as a secret in whatever system runs the playbook (e.g. a GitHub Actions secret when using `deploy-shared.yml`), not in plain configuration. When the token is present, the playbook exports it as `GH_TOKEN` for `gh release download`. +Each app in a target's `apps` mapping lists its own `env_refs` — release refs the same shape as app bundles, with no default asset name (every entry must specify an explicit `:asset-name` suffix, since there's no single obvious default under a per-app model). `deploy/deploy.py` downloads them still encrypted and checks for key collisions from the ciphertext (SOPS's dotenv output only encrypts values, so key names are readable without decryption) — scoped to that one app's own sources, not across apps, since each app ends up with its own separate `.env`. Decryption itself happens on the runner too, using the target's private age key (`credentials.secrets.sops_age_key`, a GitHub Secret) - the target host never holds this key and never runs `sops`. `apps` moved off the vault schema onto the target, and vaults moved from one-per-target to one-per-app, so that a vault can declare its output env var names directly (`HTTP_PORT`, not `TRAEFIK_HTTP_PORT`) without an implicit prefix-strip happening anywhere between the vault and the app's `.env`. ## Notable App Configurations -- **traefik**: Entry point, must be started first, uses external network -- **watchtower**: Infrastructure app — recommended on every server. Handles automatic image updates for apps that opt in via the `com.centurylinklabs.watchtower.enable=true` label. `restart.sh` automatically skips any app with this label, so Watchtower is their sole lifecycle manager. -- Apps with databases include postgres.yml and create app-specific database named `${APP_NAME}` -- Config templates use `envsubst` - variables must be shell-compatible (`${VAR}` syntax) +- **traefik**: Entry point, uses external network. Note: currently carries the Watchtower label (see below) - a target that doesn't also run a `watchtower` container (hawkeye doesn't, as of this writing) would never get `docker compose up` run for it by the automated path. Known gap, not yet resolved. +- **watchtower**: Infrastructure app — recommended on every server. Handles automatic image updates for apps that opt in via the `com.centurylinklabs.watchtower.enable=true` label. `deploy/deploy.py` skips running `docker compose pull`/`up` for any app with this label entirely, so Watchtower is their sole lifecycle manager. +- Apps with databases include a versioned template (e.g. `postgres-18.yml`) and create app-specific database named `${APP_NAME}` +- Config templates use `envsubst`-equivalent substitution (`deploy/render.py`) - variables must be shell-compatible (`${VAR}` syntax) ## Watchtower-managed Apps -Apps that carry the `com.centurylinklabs.watchtower.enable=true` label are updated automatically by Watchtower and are **skipped by `deploy.sh` after env regeneration**. The label in the compose file is the single source of truth — no separate skip list exists. +Apps that carry the `com.centurylinklabs.watchtower.enable=true` label are updated automatically by Watchtower and are **skipped by `deploy/deploy.py`** - their `.env`/config still gets pushed on every deploy, but `docker compose pull`/`up` is never run for them. The label in the compose file is the single source of truth — no separate skip list exists. Currently opted in: `traefik`, `semaphore`, `watchtower` itself. @@ -437,6 +365,6 @@ Currently opted in: `traefik`, `semaphore`, `watchtower` itself. - Template files are located in: `apps/{app}/config/*.template.*` - Generated files are created in: `apps-data/{app}/config/` -- The `up.sh` script automatically processes templates using `envsubst` and outputs to `apps-data/` -- Editing generated files directly will result in lost changes on next restart -- Always modify templates, then run `./restart.sh {app}` to regenerate +- `deploy/render.py` processes templates on the GitHub Actions runner during deploy and pushes the rendered result directly - nothing renders on the host +- Editing generated files directly will result in lost changes on the next deploy +- Always modify templates, then re-deploy to regenerate diff --git a/README.md b/README.md index 11345cf..633e727 100644 --- a/README.md +++ b/README.md @@ -1,119 +1,46 @@ # Flightdeck - Core Self-Hosted Application Runtime -A Docker-based orchestration system for deploying core self-hosted services, with optional extra application catalogs. Built with Traefik reverse proxy, automatic SSL certificate management, GitHub Release bundles, and a unified command-line interface. +A Docker-based orchestration system for deploying core self-hosted services, with optional extra application catalogs. Built with Traefik reverse proxy, automatic SSL certificate management, and GitHub Actions-driven deployment. ## 🎯 Key Features - **Core Application Set** - Essential services for routing, auth, monitoring, automation, database access, error tracking, and analytics -- **Extra Application Bundles** - Optional release app catalogs can be merged during Ansible deploy +- **Extra Application Bundles** - Optional release app catalogs can be merged during deploy - **Traefik Reverse Proxy** - Automatic routing, SSL/TLS termination, and certificate management - **Automatic SSL Certificates** - Support for Cloudflare DNS and Let's Encrypt HTTP challenges - **Modular Architecture** - Reusable docker-compose components for easy maintenance and scaling -- **Environment-based Configuration** - Three-tier configuration cascade for flexibility +- **Vault-based Configuration** - Each app's env is declared, encrypted, decrypted, and rendered per app - no server-side secrets handling - **Persistent Data Management** - Organized storage with automatic backup-friendly structure - **Database Integration** - PostgreSQL, Redis, MongoDB, TimescaleDB pre-configured - **Health Checks** - Built-in health monitoring for all services -- **CI/CD Ready** - GitHub Actions workflow for automatic deployment via Tailscale +- **Push-based Deploy** - GitHub Actions resolves, decrypts, and renders everything on the runner, then pushes a finished result over SSH -## 📋 Requirements +## 📋 Target Server Requirements + +There is no manual administration flow - target servers are never logged into to run commands, and there is no local quick-start. Every deploy goes through target/vault manifests and GitHub Actions. A target server needs only: - **Docker** >= 20.10 - **Docker Compose** >= 2.0 -- **Linux/macOS/WSL2** (Windows Subsystem for Linux 2) -- **2GB+ RAM** (recommended 4GB+ for production) -- **10GB+ Storage** (depending on applications and data) - -## 🚀 Quick Start - -### 1. Clone and Initialize - -```bash -git clone https://github.com/rubykatzen/flightdeck.git flightdeck -cd flightdeck -./up.sh -``` - -This will: - -- Copy `.env.example` to `.env` -- Create `apps-data/` directory structure -- Create Docker networks -- Set up Traefik SSL configuration - -### Release Bundle - -Merging the [Release Please](https://github.com/googleapis/release-please) release PR tags `main` and publishes a deployable project bundle as a GitHub Release asset: - -```text -rubykatzen/flightdeck@v1.2.3 -rubykatzen/flightdeck@latest -``` - -In deploy refs, `@latest` is resolved through GitHub's latest release API. It is not a mutable `latest` tag or release. - -The release asset is `flightdeck.zip` with the compose files and helper scripts, but not runtime state such as `.env`, `apps-data/`, or `backups/`. - -Download and unpack a bundle: - -```bash -tag="$(gh release view --repo rubykatzen/flightdeck --json tagName --jq .tagName)" -gh release download "$tag" --repo rubykatzen/flightdeck --pattern flightdeck.zip -unzip -q flightdeck.zip -d /opt/flightdeck/releases/latest -``` - -### 2. Configure Environment - -Edit `.env` with your settings: +- SSH access for the deploy key configured in that target's `credentials` -```bash -# Domain configuration -APPS_DOMAIN=... - -# SSL/TLS Configuration -APPS_CERTIFICATE_RESOLVER=... -APPS_CLOUDFLARE_DNS_API_TOKEN=... +Nothing else - no `sops`, no age key, no `gh`, no flightdeck scripts of any kind. All of that runs on the GitHub Actions runner instead; see "Automated Deploy" below. -# Database -APPS_DATABASE_PASSWORD=... +## 🚀 Automated Deploy -# System -APPS_TIMEZONE=... -``` +Deployment goes through [`deploy-shared.yml`](.github/workflows/deploy-shared.yml) (documented in the GitHub Actions section below), a reusable workflow wrapping [`deploy/deploy.py`](deploy/deploy.py) behind plain deploy vocabulary — `hosts`, `app-refs`, `apps`. -### Automated Deploy +The deploy is push-based and runs entirely on the GitHub Actions runner: -Deployment to a remote server goes through [`deploy-shared.yml`](.github/workflows/deploy-shared.yml) (documented in the GitHub Actions section below), a reusable workflow wrapping `ansible/deploy.yml` behind plain deploy vocabulary — `hosts`, `app-ref`, `env-refs`, `app-refs`, `apps`. `ansible/deploy.yml` is an implementation detail: callers of `deploy-shared.yml` never invoke `ansible-playbook` or write `flightdeck_*` `-e` variables themselves. +1. Resolve and download every release ref (app bundles, each app's encrypted env sources). +2. Merge the app bundles into a release tree. +3. Check each app's env sources for key collisions from the still-encrypted ciphertext (SOPS's dotenv output only encrypts values, so key names are readable without decryption) — scoped to that app's own sources, not across apps. +4. Decrypt each app's env with the target's private SOPS age key (a GitHub Secret) and write it straight into that app's `.env` in the release tree. +5. Render that app's config templates (`config/*.template.*`) using the decrypted values — the same substitution `envsubst` does, run here instead of on the host. +6. Push the finished release (real `.env`, already-rendered config) to each host over SSH, switch the `current` symlink, and run `docker compose pull && docker compose up -d` per app. -What gets deployed — which app bundles, which encrypted env sources, which apps actually run — is configured declaratively per target, not passed as ad-hoc flags; see "Vaults And Targets" below for the manifest format, including how multiple `app_refs`/`env_refs` entries merge (env merging fails loud on any key collision across sources, including against the synthesized `APPS` line). +Nothing decrypts or renders on the target host. The only thing it ever receives is a finished, ready-to-run Docker Compose project per app. -Target servers need Docker, Docker Compose, GitHub CLI (`gh`), SOPS, and the server-local age key. - -For private GitHub Releases, `deploy-shared.yml` passes a token through as `FLIGHTDECK_GITHUB_TOKEN`; the playbook exports it as `GH_TOKEN` for `gh release download`. Public releases don't need this. - -### 3. Select Applications - -Edit `.env` and set `APPS` to a comma-separated app list: - -```bash -APPS=traefik,gatus,beszel,semaphore,rybbit -``` - -### 4. Start Applications - -```bash -# Start all configured apps (Traefik must be first) -./up.sh - -# Or start specific apps -./up.sh traefik gatus rybbit - -# View logs -./logs.sh gatus -``` - -All apps will be accessible at `https://{app-name}.{APPS_DOMAIN}` - -> **Tip**: To allow per-server overrides for specific variables, declare fallback syntax in the app's `docker-compose.yml`: `${MYAPP_APPS_DOMAIN:-${APPS_DOMAIN}}`. Set `MYAPP_APPS_DOMAIN` in the server's `.env` to override for that app only. +What gets deployed — which app bundles, which apps actually run, and which encrypted env sources feed each one — is configured declaratively per target; see "Vaults And Targets" below for the manifest format. ## 📁 Project Structure @@ -123,26 +50,29 @@ flightdeck/ │ ├── traefik/ # Reverse proxy & SSL │ ├── common.yml # Shared service definitions │ ├── networks.yml # Network configuration -│ ├── postgres.yml # PostgreSQL template -│ ├── redis.yml # Redis template -│ ├── mongodb.yml # MongoDB template +│ ├── postgres-17.yml, postgres-18.yml # PostgreSQL templates +│ ├── redis-7.yml, redis-8.yml # Redis templates +│ ├── mongodb-8.yml, mysql-8.yml # More database templates +│ ├── clickhouse-25.4.yml, clickhouse-26.5.yml, timescale-17.yml, paradedb-17.yml, pgvector-17.yml # Analytics/search-oriented database templates +│ ├── gotenberg-8.yml # Document conversion template │ └── {app-name}/ # Each app directory -│ ├── .env # Generated app-scoped variables │ ├── docker-compose.yml # App configuration │ └── config/ # Optional config templates │ -├── apps-data/ # Persistent data (git-ignored) +├── apps-data/ # Persistent data on the target host, not in this repo │ ├── traefik/ # SSL certificates │ ├── postgres/ # PostgreSQL data -│ └── {app-name}/ # Each app's data -│ └── ... # App data directories +│ └── {app-name}/ # Each app's data + rendered config │ -├── backups/ # Backup archives (git-ignored) -├── ansible/ -│ └── deploy.yml # Deploy published bundle and encrypted env +├── deploy/ +│ ├── deploy.py # Push-based deploy entrypoint (runs on the CI runner) +│ ├── resolve.py # owner/repo@tag[:asset] release ref resolution/download +│ ├── collisions.py # Ciphertext-based env key collision detection +│ ├── vault.py # SOPS decryption +│ └── render.py # envsubst-equivalent config template rendering ├── .github/ │ ├── actions/ -│ │ ├── build-bundle/ # Build and upload the machinery bundle +│ │ ├── build-bundle/ # Build and upload a zip bundle from given paths │ │ ├── build-apps-bundle/ # Build and upload an apps/ catalog bundle │ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release │ │ └── load-yaml-matrix/ # Read a directory of YAML manifests into a workflow matrix @@ -150,69 +80,10 @@ flightdeck/ │ ├── deploy-shared.yml # Reusable deployment workflow │ └── release.yml # Release Please + publish Flightdeck assets │ -├── vaults/ # Encrypted env asset configurations -├── targets/ # Deployment targets -├── .env # All server configuration incl. APPS list (git-ignored) -├── .env.example # Configuration template -│ -├── up.sh # Start applications -├── down.sh # Stop applications -├── restart.sh # Restart applications -├── logs.sh # View application logs -└── backup.sh # Backup app data +├── vaults/ # Encrypted env asset configurations, one per app +└── targets/ # Deployment targets ``` -## 🎮 Common Commands - -### Start Applications - -```bash -# Start all apps defined in APPS (pulls latest images automatically) -./up.sh - -# Start specific apps -./up.sh traefik gatus rybbit -``` - -### Stop Applications - -```bash -# Stop all apps -./down.sh - -# Stop specific apps -./down.sh gatus rybbit -``` - -### Restart Applications - -```bash -# Restart all apps -./restart.sh - -# Restart specific apps -./restart.sh gatus rybbit -``` - -### View Logs - -```bash -# View logs for specific app (requires single app name) -./logs.sh gatus - -# View logs with timestamps -./logs.sh rybbit -``` - -### Backup Applications - -```bash -# Backup all apps from a remote server -./backup.sh user@server.com -``` - -The script stops each app one at a time, creates a zip archive, restarts it, then downloads the archive to `backups/`. Files are named `{server}-{app}-{datetime}.zip`. - ## 📦 Core Applications | Name | Purpose | @@ -227,49 +98,22 @@ The script stops each app one at a time, creates a zip archive, restarts it, the | **databasus** | Database management UI | | **rybbit** | Web analytics | -This catalog is itself published as its own release asset (`flightdeck-apps.zip`), merged at deploy time like any other entry in `flightdeck_app_refs`. Additional apps can live in any other repo's own `apps/`-shaped catalog, published the same way, and merged in by listing its ref alongside flightdeck's own. +This catalog is itself published as its own release asset (`flightdeck-apps.zip`), merged at deploy time like any other entry in `app_refs`. Additional apps can live in any other repo's own `apps/`-shaped catalog, published the same way, and merged in by listing its ref alongside flightdeck's own. ## ⚙️ Configuration ### Environment Variables -Variables use a scoped env model: - -**1. Server env (`/.env`)**: - -Contains all variables for this server: shared `APPS_*`, per-app variables, and the comma-separated `APPS` list. Deployed by Ansible from an encrypted secrets release asset. - -```bash -APPS # Comma-separated apps to deploy on this server -APPS_DOMAIN # Base domain (required) -APPS_CERTIFICATE_RESOLVER # letsencrypt or cloudflare -APPS_CLOUDFLARE_DNS_API_TOKEN # If using Cloudflare DNS -APPS_DATABASE_PASSWORD # PostgreSQL/MySQL password -APPS_KEY_HEX_16 # 16-byte hex key for apps -APPS_KEY_HEX_32 # 32-byte hex key for apps -APPS_KEY_HEX_64 # 64-byte hex key for apps -APPS_TIMEZONE # System timezone (UTC, etc.) -``` - -**2. Generated app env (`/apps/{app}/.env`)**: - -Auto-generated by `generate-env.sh` — do not edit. Contains `APP_NAME`, all shared `APPS_*` variables, and only variables beginning with the normalized app prefix. - -Examples: +Every app's env comes from its own vault(s), declared in that target's manifest (see "Vaults And Targets" below). A vault declares the exact final variable names an app receives, mapped to GitHub Secret/Variable names - there is no server-side prefix filtering or shared root env file. Two apps' vaults can share a source secret (e.g. both mapping `APPS_DOMAIN`) without conflict, since each app ends up with its own separate `.env`. -- `twofauth` receives `APPS_*` and `TWOFAUTH_*` -- `beszel-agent` receives `APPS_*` and `BESZEL_AGENT_*` - -An app does not receive another app's env variables. - -**Per-app and per-server overrides** are declared directly in each app's `docker-compose.yml` using bash fallback syntax: +**Per-app overrides** are declared directly in each app's `docker-compose.yml` using bash fallback syntax: ```yaml # App-specific override, falls back to server-wide value SOME_PATH: ${MYAPP_SOME_PATH:-${APPS_SOME_PATH}} ``` -Set `MYAPP_SOME_PATH` in the server's `.env` to override for that app only. App prefixes are the uppercased app directory with hyphens replaced by underscores. See `AGENTS.md` for the full naming convention. +To use the override for a given deploy, that app's own vault sets `MYAPP_SOME_PATH` in its `env:` mapping; to fall back to the shared value, the vault just omits it and relies on `APPS_SOME_PATH` alone. App prefixes are the uppercased app directory with hyphens replaced by underscores. See `AGENTS.md` for the full naming convention. ### Network Architecture @@ -278,7 +122,7 @@ Set `MYAPP_SOME_PATH` in the server's `.env` to override for that app only. App - **databases** - Dedicated network for database services (PostgreSQL, Redis, MongoDB) - **mcp** - External network for MCP services consumed by MetaMCP -Apps are automatically connected to appropriate networks based on their needs. +`traefik`, `databases`, and `mcp` are created on the target host by `deploy/deploy.py` (derived from `apps/networks.yml`'s `external: true` entries); `internal` is created by Docker Compose itself. ## 🆕 Adding a New Application @@ -314,33 +158,24 @@ services: - ../../apps-data/${APP_NAME}/data:/data ``` -### Step 3: Add to `.env` - -Add your app to `APPS` in `.env`: - -```bash -APPS=traefik,myapp -``` +**Important**: Always use the `x-environment` anchor pattern for environment variables. This ensures consistency and reduces duplication. -### Step 4: Start the App +### Step 3: Wire It Into a Target -```bash -./up.sh myapp -``` - -**Important**: Always use the `x-environment` anchor pattern for environment variables. This ensures consistency and reduces duplication. +Add the app to whichever target's `apps` mapping should run it, and give it a vault declaring the env it needs (`MY_VALUE` in the example above) — see "Vaults And Targets" below. There is no local way to run an app outside of a real deploy; verify a new app definition by deploying it to a real (even if disposable) target. ## 🔍 Troubleshooting +Since there's no manual administration path, all of the following happen by SSHing into the target host directly, for debugging only: + ### Container Won't Start ```bash # Check logs -./logs.sh app-name +cd apps/app-name && docker compose logs -f # Validate docker-compose configuration -docker compose --env-file ./apps/app-name/.env --env-file .env \ - -f ./apps/app-name/docker-compose.yml config +cd apps/app-name && docker compose config # Check network connectivity docker network ls @@ -351,34 +186,31 @@ docker network inspect traefik ```bash # Check Traefik logs -./logs.sh traefik +cd apps/traefik && docker compose logs -f # Verify ACME certificate file ls -la apps-data/traefik/acme.json # Ensure correct permissions chmod 600 apps-data/traefik/acme.json - -# For Cloudflare issues, verify API token is set in .env -grep APPS_CLOUDFLARE_DNS_API_TOKEN .env ``` ### Application Not Accessible 1. Verify app is running: `docker ps | grep app-name` -2. Check app logs: `./logs.sh app-name` -3. Check Traefik logs: `./logs.sh traefik` +2. Check app logs: `cd apps/app-name && docker compose logs -f` +3. Check Traefik logs: `cd apps/traefik && docker compose logs -f` 4. Verify DNS resolves: `nslookup app-name.domain.com` 5. Test internal connectivity: `docker exec -it traefik wget -q --spider http://app-name` ## 🔐 Security Best Practices -1. **Change Default Credentials** - Update passwords in `.env` and app configurations +1. **Change Default Credentials** - Update vault-sourced secrets and re-deploy 2. **Use Strong Passwords** - Generate with: `openssl rand -base64 32` -3. **Keep Images Updated** - Run `./restart.sh` regularly (`up.sh` pulls latest images automatically) +3. **Keep Images Updated** - Watchtower-managed apps update automatically; others get the latest image on every deploy (`docker compose pull` runs before `up`) 4. **Restrict Network Access** - Use firewall rules to limit access to Traefik ports (80, 443) 5. **Enable HTTPS** - Always use HTTPS, never expose HTTP to internet -6. **Backup Data** - Regularly backup `apps-data/` directory +6. **Backup Data** - Regularly back up `apps-data/` (backup automation is a separate, not-yet-decided piece of tooling) 7. **Monitor Logs** - Review logs regularly for errors and unauthorized access attempts 8. **Update Dependencies** - Check for updates: `docker pull app:latest` @@ -394,7 +226,7 @@ grep APPS_CLOUDFLARE_DNS_API_TOKEN .env Contributions are welcome! To add a new application: 1. Follow the "Adding a New Application" section -2. Test thoroughly with `./up.sh app-name` +2. Verify by deploying it to a real target 3. Document any special requirements 4. Submit a pull request with the new app configuration @@ -404,7 +236,7 @@ If you're evaluating alternatives, these projects solve a similar problem from d | Service | Website | Focus | Service Templates | |------|---------|---------|---------| -| **flightdeck** | This repository | Git-based Docker Compose stack with reusable templates and shell scripts | [apps](./apps/) | +| **flightdeck** | This repository | Git-based Docker Compose stack with reusable templates, deployed via GitHub Actions | [apps](./apps/) | | **Dokploy** | [dokploy.com](https://dokploy.com) | PaaS-style deployment panel for apps, databases, and containers | [Dokploy/templates/blueprints](https://github.com/Dokploy/templates/tree/canary/blueprints) | | **Runtipi** | [runtipi.io](https://runtipi.io) | Beginner-friendly self-hosted app store and dashboard | [runtipi/runtipi-appstore/apps](https://github.com/runtipi/runtipi-appstore/tree/master/apps) | | **Coolify** | [coolify.io](https://coolify.io) | Self-hosted Heroku/Vercel-style platform for apps, databases, and services | [coollabsio/coolify/templates/compose](https://github.com/coollabsio/coolify/tree/v4.x/templates/compose) | @@ -417,12 +249,22 @@ This repository provides four composite actions under `.github/actions/` (`build ### Vaults And Targets -Files in `vaults/` describe encrypted env assets — pure secrets/config, no app selection. Files in `targets/` describe deployments, including the desired `apps` set. The two collections are independent; a deployment links to encrypted assets explicitly through `env_refs`. Matching filenames are a convenience, not an implicit relationship. +Files in `vaults/` describe encrypted env assets, one per app — pure secrets/config, no app selection. Files in `targets/` describe deployments, including which apps run and which vault(s) feed each one. The two collections are independent; a target links to encrypted assets explicitly through each app's own `env_refs`. Matching filenames are a convenience, not an implicit relationship. + +`vaults/mainframe-traefik.yml`: + +```yaml +asset: mainframe-traefik.sops.env +keys: + - mainframe +env: + HTTP_PORT: MAINFRAME_TRAEFIK_HTTP_PORT +``` -`vaults/mainframe.yml`: +`vaults/mainframe-rybbit.yml`: ```yaml -asset: mainframe.sops.env +asset: mainframe-rybbit.sops.env keys: - mainframe env: @@ -432,31 +274,32 @@ env: `targets/mainframe.yml`: ```yaml -flightdeck_ref: rubykatzen/flightdeck@latest -env_refs: - - owner/config@latest:mainframe.sops.env app_refs: - rubykatzen/flightdeck@latest - owner/extra-apps@latest apps: - - traefik - - rybbit + traefik: + env_refs: + - owner/config@latest:mainframe-traefik.sops.env + rybbit: + env_refs: + - owner/config@latest:mainframe-rybbit.sops.env hosts: - deploy@100.64.0.1 - deploy@100.64.0.2 path: ~/flightdeck # optional, default shown -sops_age_key_file: ~/.config/sops/age/keys.txt # optional, default shown credentials: variables: tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID secrets: ssh_private_key: DEPLOY_SSH_PRIVATE_KEY tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET + sops_age_key: MAINFRAME_AGE_PRIVATE_KEY ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `env_refs`, `app_refs`, `apps`, and `hosts` are YAML arrays. Each host uses the SSH `user@host` format. `app_refs` must list at least one app bundle — flightdeck's own `apps/` catalog is just another entry, not implicit. `env_refs` must list at least one encrypted env asset; the deploy playbook decrypts and merges all of them plus a synthesized `APPS` line built from the target's own `apps`, failing loud on any key collision across sources (including against `APPS` itself, if a vault ever tried to define it). +Credential fields contain GitHub Variable/Secret names, never credential values. `app_refs` and `hosts` are YAML arrays; `apps` is a mapping from app name to that app's own `env_refs` array. Each host uses the SSH `user@host` format. `app_refs` must list at least one app bundle — flightdeck's own `apps/` catalog is just another entry, not implicit. Each app in `apps` must list at least one `env_refs` entry; `deploy/deploy.py` decrypts and concatenates all of an app's sources into that app's own `.env` on the runner, failing loud on any key collision — but only within that one app's own sources. Two different apps' vaults sharing a key (e.g. both declaring `APPS_DOMAIN`) is expected, since each app gets a separate `.env`. `credentials.secrets.sops_age_key` names the GitHub Secret holding this target's *private* age key — the one used to decrypt its vaults, matching the public key in `keys/.pub` used to encrypt them. -`load-yaml-matrix` reads every file in `vaults/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases`/`sops-age-key-file` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. +`load-yaml-matrix` reads every file in `vaults/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. --- @@ -467,7 +310,7 @@ Renders an encryption config from GitHub Secrets/Variables, encrypts it with SOP ```yaml - uses: rubykatzen/flightdeck/.github/actions/encrypt-env@main with: - manifest: vaults/mainframe.yml # required + manifest: vaults/mainframe-traefik.yml # required keys-directory: keys # default: keys release-tag: latest # required, must already exist release-repo: "" # default: current repository @@ -482,11 +325,11 @@ Requires `contents: write` permission on the calling job. **Manifest format:** ```yaml -asset: mainframe.sops.env +asset: mainframe-traefik.sops.env keys: - mainframe env: - APPS_DOMAIN: APPS_DOMAIN # output name: GitHub Secret/Variable name + HTTP_PORT: MAINFRAME_TRAEFIK_HTTP_PORT # output name: GitHub Secret/Variable name ``` Secrets take precedence over Variables when both contain the same source key. Every source key must exist or the action fails. @@ -495,7 +338,7 @@ Secrets take precedence over Variables when both contain the same source key. Ev ### `build-bundle` -Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. `paths` and `bundle-name` default to Flightdeck's own machinery bundle (everything except `apps/`, uploaded as `flightdeck.zip`) but are fully overridable. +Builds a zip archive from caller-selected paths, rejects runtime state and env files, and uploads it to an existing GitHub Release. `paths` and `bundle-name` are required — this is a generic, reusable primitive (`build-apps-bundle` below is the only current caller). ```yaml steps: @@ -504,10 +347,10 @@ steps: ref: v1.2.3 - uses: rubykatzen/flightdeck/.github/actions/build-bundle@v1.2.3 with: + paths: apps + bundle-name: flightdeck-apps.zip release-tag: v1.2.3 token: ${{ secrets.GITHUB_TOKEN }} - # paths: ... # optional, defaults to the machinery file list - # bundle-name: ... # optional, defaults to flightdeck.zip ``` Requires `contents: write` permission on the calling job. @@ -529,15 +372,15 @@ steps: token: ${{ secrets.GITHUB_TOKEN }} ``` -Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` is the default asset name a `flightdeck_app_refs` entry resolves to when it doesn't specify an explicit `:asset-name` suffix; override `bundle-name` and use that suffix when publishing under a different filename. +Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` is the default asset name an `app_refs` entry resolves to when it doesn't specify an explicit `:asset-name` suffix; override `bundle-name` and use that suffix when publishing under a different filename. --- ### `deploy-shared.yml` -Runs [`ansible/deploy.yml`](ansible/deploy.yml) from this repository against the caller-supplied hosts. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, etc.) — this repository does not hold any deploy secrets itself. `env-refs` entries typically reference that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. +Runs [`deploy/deploy.py`](deploy/deploy.py) from this repository against the caller-supplied hosts. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, the age private key, etc.) — this repository does not hold any deploy secrets itself. `apps..env_refs` entries typically reference that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. -The interface is plain deploy vocabulary, not Ansible's — callers never see `flightdeck_*` variable names or hand-write `-e` JSON; the workflow builds that internally. +The interface is plain deploy vocabulary — callers never see `deploy.py`'s internals or hand-write its JSON config; the workflow builds that internally and pipes it to `python3 deploy/deploy.py` on stdin. The runner resolves and downloads every ref, decrypts and renders each app's env and config, merges the release, and pushes the finished result to each host over SSH — see "Automated Deploy" above for the full sequence. Tailscale is optional, not a dependency of this workflow: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the hosts, or reaches them some other way. @@ -547,21 +390,19 @@ jobs: uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v1.2.3 with: hosts: '["deploy@100.64.0.1", "deploy@100.64.0.2"]' # required JSON array - app-ref: rubykatzen/flightdeck@latest # required full release ref - env-refs: '["${{ github.repository }}@latest:.sops.env"]' # required non-empty JSON array app-refs: '["rubykatzen/flightdeck@latest"]' # required non-empty JSON array - apps: '["traefik", "rybbit"]' # required non-empty JSON array + apps: '{"traefik": {"env_refs": ["${{ github.repository }}@latest:mainframe-traefik.sops.env"]}}' # required non-empty JSON object # path: ~/flightdeck # optional, default shown # keep-releases: 5 # optional, default shown - # sops-age-key-file: /home/deploy/.config/sops/age/keys.txt # optional, default: ~/.config/sops/age/keys.txt for `user` tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) tailscale-tags: tag:ci # default: tag:ci secrets: ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + sops-age-key: ${{ secrets.MAINFRAME_AGE_PRIVATE_KEY }} tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # optional, required only if tailscale-oauth-client-id is set ``` -The `@v1.2.3` pin on the `uses:` line only controls which ref runs the playbook mechanism itself. `app-ref` is separate and required - it is the full release ref for the bundle the playbook downloads and deploys, and does not have to match the workflow pin. +The `@v1.2.3` pin on the `uses:` line only controls which ref runs `deploy/deploy.py` itself. `app-refs` entries are separate and don't have to match the workflow pin. ## 📝 License diff --git a/ansible.cfg b/ansible.cfg deleted file mode 100644 index 44ef0e0..0000000 --- a/ansible.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[defaults] -interpreter_python = auto_silent diff --git a/ansible/deploy.yml b/ansible/deploy.yml deleted file mode 100644 index bf465c4..0000000 --- a/ansible/deploy.yml +++ /dev/null @@ -1,302 +0,0 @@ -- name: Deploy Flightdeck - hosts: all - become: true - tasks: - - name: Read Flightdeck GitHub token - set_fact: - flightdeck_github_token: "{{ lookup('env', 'FLIGHTDECK_GITHUB_TOKEN') }}" - no_log: true - - name: Require Flightdeck deploy variables - assert: - that: - - flightdeck_app_ref is defined and flightdeck_app_ref | length > 0 - - flightdeck_env_refs is defined and flightdeck_env_refs | length > 0 - - flightdeck_app_refs is defined and flightdeck_app_refs | length > 0 - - flightdeck_apps is defined and flightdeck_apps | length > 0 - - flightdeck_path is defined and flightdeck_path | length > 0 - - flightdeck_keep_releases is defined - - flightdeck_sops_age_key_file is defined and flightdeck_sops_age_key_file | length > 0 - fail_msg: "Set flightdeck_app_ref, flightdeck_env_refs, flightdeck_app_refs, flightdeck_apps, flightdeck_path, flightdeck_keep_releases, and flightdeck_sops_age_key_file" - - name: Resolve Flightdeck user paths - set_fact: - flightdeck_user_home: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}" - - name: Expand Flightdeck user paths - set_fact: - flightdeck_path: "{{ flightdeck_path | regex_replace('^~', flightdeck_user_home) }}" - flightdeck_sops_age_key_file: "{{ flightdeck_sops_age_key_file | regex_replace('^~', flightdeck_user_home) }}" - - name: Apply Flightdeck paths - set_fact: - flightdeck_release_name: "{{ ansible_facts['date_time'].iso8601_basic_short }}" - flightdeck_release_path: "{{ flightdeck_path }}/releases/{{ ansible_facts['date_time'].iso8601_basic_short }}" - flightdeck_shared_path: "{{ flightdeck_path }}/shared" - flightdeck_current_path: "{{ flightdeck_path }}/current" - - name: Ensure Flightdeck directories - file: - path: "{{ item }}" - state: directory - mode: "0755" - loop: - - "{{ flightdeck_path }}" - - "{{ flightdeck_path }}/releases" - - "{{ flightdeck_shared_path }}" - - block: - - name: Create app package pull directory - tempfile: - state: directory - suffix: flightdeck-package - register: flightdeck_app_pull - - name: Pull Flightdeck package - shell: | - set -euo pipefail - ref={{ flightdeck_app_ref | quote }} - out={{ flightdeck_app_pull.path | quote }} - asset=flightdeck.zip - repo="${ref%@*}" - tag="${ref#*@}" - if [ "$repo" = "$ref" ] || [ -z "$repo" ] || [ -z "$tag" ]; then - echo "Invalid release ref: $ref. Expected owner/repo@tag or owner/repo@tag:asset" >&2 - exit 1 - fi - if [ "$tag" != "${tag%%:*}" ]; then - asset="${tag#*:}" - tag="${tag%%:*}" - fi - if [ -n "${FLIGHTDECK_GITHUB_TOKEN:-}" ]; then - export GH_TOKEN="$FLIGHTDECK_GITHUB_TOKEN" - fi - if [ "$tag" = "latest" ]; then - resolved_tag="$(gh release view --repo "$repo" --json tagName --jq .tagName)" - if [ -z "$resolved_tag" ] || [ "$resolved_tag" = "null" ]; then - echo "Could not resolve GitHub latest release for $repo" >&2 - exit 1 - fi - echo "Resolved $repo@latest to $repo@$resolved_tag" - tag="$resolved_tag" - fi - gh release download "$tag" --repo "$repo" --pattern "$asset" --dir "$out" - test -f "$out/$asset" - if [ "$asset" != "flightdeck.zip" ]; then - cp "$out/$asset" "$out/flightdeck.zip" - fi - args: - executable: /bin/bash - environment: - FLIGHTDECK_GITHUB_TOKEN: "{{ flightdeck_github_token }}" - - name: Require Flightdeck bundle - stat: - path: "{{ flightdeck_app_pull.path }}/flightdeck.zip" - register: flightdeck_bundle - - name: Fail when Flightdeck bundle is missing - fail: - msg: "flightdeck.zip was not found in {{ flightdeck_app_ref }}" - when: not flightdeck_bundle.stat.exists - - name: Recreate release directory - file: - path: "{{ flightdeck_release_path }}" - state: absent - - name: Ensure release directory - file: - path: "{{ flightdeck_release_path }}" - state: directory - mode: "0755" - - name: Extract Flightdeck release - unarchive: - src: "{{ flightdeck_app_pull.path }}/flightdeck.zip" - dest: "{{ flightdeck_release_path }}" - remote_src: true - - name: Ensure release apps directory - file: - path: "{{ flightdeck_release_path }}/apps" - state: directory - mode: "0755" - - name: Create app packages pull directory - tempfile: - state: directory - suffix: flightdeck-app-packages - register: flightdeck_app_packages_pull - - name: Pull and merge app packages - shell: | - set -euo pipefail - packages_root={{ flightdeck_app_packages_pull.path | quote }} - release_apps={{ (flightdeck_release_path + '/apps') | quote }} - download_release_ref() { - ref="$1" - out="$2" - default_asset="$3" - repo="${ref%@*}" - tag="${ref#*@}" - asset="$default_asset" - if [ "$repo" = "$ref" ] || [ -z "$repo" ] || [ -z "$tag" ]; then - echo "Invalid release ref: $ref. Expected owner/repo@tag or owner/repo@tag:asset" >&2 - exit 1 - fi - if [ "$tag" != "${tag%%:*}" ]; then - asset="${tag#*:}" - tag="${tag%%:*}" - fi - if [ "$tag" = "latest" ]; then - resolved_tag="$(gh release view --repo "$repo" --json tagName --jq .tagName)" - if [ -z "$resolved_tag" ] || [ "$resolved_tag" = "null" ]; then - echo "Could not resolve GitHub latest release for $repo" >&2 - exit 1 - fi - echo "Resolved $repo@latest to $repo@$resolved_tag" >&2 - tag="$resolved_tag" - fi - gh release download "$tag" --repo "$repo" --pattern "$asset" --dir "$out" - test -f "$out/$asset" - echo "$out/$asset" - } - if [ -n "${FLIGHTDECK_GITHUB_TOKEN:-}" ]; then - export GH_TOKEN="$FLIGHTDECK_GITHUB_TOKEN" - fi - {% for ref in flightdeck_app_refs %} - package_dir="$packages_root/{{ loop.index }}" - mkdir -p "$package_dir/pull" "$package_dir/extract" - bundle="$(download_release_ref {{ ref | quote }} "$package_dir/pull" flightdeck-apps.zip)" - unzip "$bundle" -d "$package_dir/extract" - if [ ! -d "$package_dir/extract/apps" ]; then - echo "Package {{ ref }} does not contain apps/" >&2 - exit 1 - fi - for app_path in "$package_dir/extract/apps"/*; do - [ -d "$app_path" ] || continue - app="$(basename "$app_path")" - if [ -e "$release_apps/$app" ]; then - echo "App conflicts with an existing app: $app" >&2 - exit 1 - fi - cp -a "$app_path" "$release_apps/" - done - {% endfor %} - args: - executable: /bin/bash - environment: - FLIGHTDECK_GITHUB_TOKEN: "{{ flightdeck_github_token }}" - - name: Create env packages pull directory - tempfile: - state: directory - suffix: flightdeck-env-packages - register: flightdeck_env_packages_pull - - name: Pull, decrypt, and merge env packages - shell: | - set -euo pipefail - packages_root={{ flightdeck_env_packages_pull.path | quote }} - merged={{ (flightdeck_shared_path + '/.env.tmp') | quote }} - seen_keys_file="$packages_root/seen-keys" - : > "$seen_keys_file" - : > "$merged" - add_line() { - key="${1%%=*}" - if grep -qxF "$key" "$seen_keys_file"; then - echo "Env key conflicts with an existing source: $key" >&2 - exit 1 - fi - echo "$key" >> "$seen_keys_file" - printf '%s\n' "$1" >> "$merged" - } - add_line {{ ('APPS=' + (flightdeck_apps | join(','))) | quote }} - if [ -n "${FLIGHTDECK_GITHUB_TOKEN:-}" ]; then - export GH_TOKEN="$FLIGHTDECK_GITHUB_TOKEN" - fi - {% for ref in flightdeck_env_refs %} - pkg_dir="$packages_root/{{ loop.index }}" - mkdir -p "$pkg_dir" - ref={{ ref | quote }} - asset= - repo="${ref%@*}" - tag="${ref#*@}" - if [ "$repo" = "$ref" ] || [ -z "$repo" ] || [ -z "$tag" ]; then - echo "Invalid release ref: $ref. Expected owner/repo@tag or owner/repo@tag:asset" >&2 - exit 1 - fi - if [ "$tag" != "${tag%%:*}" ]; then - asset="${tag#*:}" - tag="${tag%%:*}" - fi - if [ "$tag" = "latest" ]; then - resolved_tag="$(gh release view --repo "$repo" --json tagName --jq .tagName)" - if [ -z "$resolved_tag" ] || [ "$resolved_tag" = "null" ]; then - echo "Could not resolve GitHub latest release for $repo" >&2 - exit 1 - fi - echo "Resolved $repo@latest to $repo@$resolved_tag" >&2 - tag="$resolved_tag" - fi - if [ -z "$asset" ]; then - asset="$tag.sops.env" - fi - gh release download "$tag" --repo "$repo" --pattern "$asset" --dir "$pkg_dir" - test -f "$pkg_dir/$asset" - plain="$pkg_dir/plain.env" - SOPS_AGE_KEY_FILE={{ flightdeck_sops_age_key_file | quote }} sops decrypt "$pkg_dir/$asset" > "$plain" - while IFS= read -r line; do - [ -n "$line" ] || continue - add_line "$line" - done < "$plain" - {% endfor %} - args: - executable: /bin/bash - environment: - FLIGHTDECK_GITHUB_TOKEN: "{{ flightdeck_github_token }}" - no_log: true - - name: Set env file permissions - file: - path: "{{ flightdeck_shared_path }}/.env.tmp" - mode: "0600" - - name: Publish env file - command: - cmd: "mv {{ flightdeck_shared_path }}/.env.tmp {{ flightdeck_shared_path }}/.env" - - name: Ensure shared apps-data directory - file: - path: "{{ flightdeck_shared_path }}/apps-data" - state: directory - mode: "0755" - - name: Link shared files into release - file: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - state: link - force: true - loop: - - src: "{{ flightdeck_shared_path }}/.env" - dest: "{{ flightdeck_release_path }}/.env" - - src: "{{ flightdeck_shared_path }}/apps-data" - dest: "{{ flightdeck_release_path }}/apps-data" - - name: Switch current release - file: - src: "{{ flightdeck_release_path }}" - dest: "{{ flightdeck_current_path }}" - state: link - force: true - - name: Deploy Flightdeck - command: - cmd: ./deploy.sh - chdir: "{{ flightdeck_current_path }}" - - name: Find Flightdeck releases - find: - paths: "{{ flightdeck_path }}/releases" - file_type: directory - recurse: false - register: flightdeck_releases - - name: Remove old Flightdeck releases - file: - path: "{{ item.path }}" - state: absent - loop: "{{ (flightdeck_releases.files | sort(attribute='mtime', reverse=true))[flightdeck_keep_releases | int:] }}" - always: - - name: Remove app package pull directory - file: - path: "{{ flightdeck_app_pull.path }}" - state: absent - when: flightdeck_app_pull is defined and flightdeck_app_pull.path is defined - - name: Remove env packages pull directory - file: - path: "{{ flightdeck_env_packages_pull.path }}" - state: absent - when: flightdeck_env_packages_pull is defined and flightdeck_env_packages_pull.path is defined - - name: Remove app packages pull directory - file: - path: "{{ flightdeck_app_packages_pull.path }}" - state: absent - when: flightdeck_app_packages_pull is defined and flightdeck_app_packages_pull.path is defined diff --git a/apps/traefik/docker-compose.yml b/apps/traefik/docker-compose.yml index 9dbb647..f6ec909 100644 --- a/apps/traefik/docker-compose.yml +++ b/apps/traefik/docker-compose.yml @@ -7,8 +7,8 @@ services: labels: - "com.centurylinklabs.watchtower.enable=true" ports: - - "${TRAEFIK_HTTP_PORT}:80" - - "${TRAEFIK_HTTPS_PORT}:443" + - "${HTTP_PORT}:80" + - "${HTTPS_PORT}:443" volumes: - /etc/localtime:/etc/localtime:ro - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/backup.sh b/backup.sh deleted file mode 100755 index 04780b9..0000000 --- a/backup.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -set -e - -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -SERVER="$1" -REMOTE_APPS_DATA="~/flightdeck/apps-data" -REMOTE_PROJECT="~/flightdeck" -LOCAL_BACKUPS_DIR="$(dirname "$0")/backups" -DATETIME=$(date +"%Y%m%d_%H%M%S") -SERVER_NAME=$(echo "$SERVER" | sed 's/[^a-zA-Z0-9]/_/g') - -mkdir -p "$LOCAL_BACKUPS_DIR" - -echo "==> Getting list of app directories..." -APPS=$(ssh "$SERVER" "ls -d $REMOTE_APPS_DATA/*/ 2>/dev/null | xargs -I{} basename {}") - -if [ -z "$APPS" ]; then - echo "No app directories found in $REMOTE_APPS_DATA" - exit 0 -fi - -echo "==> Reading APPS from remote .env..." -APPS_LIST=$(ssh "$SERVER" "bash -c 'source $REMOTE_PROJECT/.env && echo \"\${APPS//,/ }\"'") - -echo "==> Found apps: $(echo "$APPS" | tr '\n' ' ')" -echo "==> Active APPS: $APPS_LIST" - -REMOTE_TMP="/tmp/flightdeck-backup-$$" -ssh "$SERVER" "mkdir -p $REMOTE_TMP" - -for APP in $APPS; do - ARCHIVE="${APP}-${DATETIME}-${SERVER_NAME}.zip" - echo "" - if echo " $APPS_LIST " | grep -qw "$APP"; then - echo "==> [$APP] Stopping container..." - ssh "$SERVER" "cd $REMOTE_PROJECT && ./down.sh $APP" > /dev/null 2>&1 - echo "==> [$APP] Creating zip archive..." - ssh "$SERVER" "cd $REMOTE_APPS_DATA && find $APP/ -not -type s | zip $REMOTE_TMP/$ARCHIVE -@ -q" - echo "==> [$APP] Starting container back..." - ssh "$SERVER" "cd $REMOTE_PROJECT && ./up.sh $APP" > /dev/null 2>&1 - else - echo "==> [$APP] Not in APPS, skipping stop/start..." - ssh "$SERVER" "cd $REMOTE_APPS_DATA && find $APP/ -not -type s | zip $REMOTE_TMP/$ARCHIVE -@ -q" - fi - echo "==> [$APP] Downloading archive..." - scp "$SERVER:$REMOTE_TMP/$ARCHIVE" "$LOCAL_BACKUPS_DIR/$ARCHIVE" - ssh "$SERVER" "rm -f $REMOTE_TMP/$ARCHIVE" - echo "==> [$APP] Done." -done - -ssh "$SERVER" "rm -rf $REMOTE_TMP" - -echo "" -echo "Backup complete. Files saved to: $LOCAL_BACKUPS_DIR" -ls -lh "$LOCAL_BACKUPS_DIR/"*"-${DATETIME}-${SERVER_NAME}.zip" 2>/dev/null || true diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 8ddeda1..0000000 --- a/deploy.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e -source "$(dirname "$0")/lib.sh" -set -a -source .env -set +a - -if [ $# -gt 0 ]; then - apps=("$@") -else - parse_apps "$APPS" -fi - -"$(dirname "$0")/generate-env.sh" "${apps[@]}" - -for app in "${apps[@]}"; do - require_app_compose "${app}" - if grep -q 'com.centurylinklabs.watchtower.enable=true' "./apps/${app}/docker-compose.yml" 2>/dev/null; then - echo "Skipping restart: ${app} (managed by Watchtower, env regenerated)" - continue - fi - "$(dirname "$0")/restart.sh" "$app" -done diff --git a/deploy/collisions.py b/deploy/collisions.py new file mode 100644 index 0000000..aac1847 --- /dev/null +++ b/deploy/collisions.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Detect duplicate env keys across an app's env_refs sources, from ciphertext. + +SOPS's dotenv output format only encrypts values, not key names +(`APPS_DOMAIN=ENC[...]`), so this needs no decryption at all - it runs in +CI, before anything is pushed to the target host, on files whose private +key CI never has access to in the first place. +""" +from pathlib import Path + + +class CollisionError(Exception): + pass + + +def extract_keys(sops_env_path): + keys = set() + for line in Path(sops_env_path).read_text().splitlines(): + if "=" not in line: + continue + key = line.split("=", 1)[0] + if not key or key.startswith("sops_"): + continue + keys.add(key) + return keys + + +def check_env_collisions(sops_env_paths): + seen = {} + for path in sops_env_paths: + for key in extract_keys(path): + if key in seen: + raise CollisionError( + f"Env key conflicts with an existing source: {key} " + f"(in {seen[key]} and {path})" + ) + seen[key] = path diff --git a/deploy/deploy.py b/deploy/deploy.py new file mode 100644 index 0000000..25c8973 --- /dev/null +++ b/deploy/deploy.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Push-based deploy: resolve, download, decrypt, and render everything +here on the runner, then push a fully finished release tree - real +plaintext `.env` files and already-rendered config - to each target host +over SSH, and run a short remote command sequence. The target host needs +nothing but Docker and Docker Compose: no sops, no age key, no gh, no +flightdeck scripts of any kind. + +Reads a JSON config from stdin (see README's "deploy-shared.yml" section +for the exact shape). +""" +import io +import json +import shlex +import shutil +import sys +import tarfile +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from zipfile import ZipFile + +import paramiko +import yaml +from collisions import check_env_collisions +from fabric import Connection +from render import render_template +from resolve import download_ref +from vault import decrypt_env, parse_dotenv + +APPS_BUNDLE_ASSET = "flightdeck-apps.zip" +WATCHTOWER_LABEL = "com.centurylinklabs.watchtower.enable=true" + + +class DeployError(Exception): + pass + + +def build_release(config, work_dir): + pull_dir = work_dir / "pull" + release_dir = work_dir / "release" + apps_dir = release_dir / "apps" + apps_dir.mkdir(parents=True) + + for index, ref in enumerate(config["app_refs"], start=1): + package_dir = pull_dir / f"apps-{index}" + bundle = download_ref(ref, package_dir / "pull", default_asset=APPS_BUNDLE_ASSET) + extract_dir = package_dir / "extract" + with ZipFile(bundle) as archive: + archive.extractall(extract_dir) + + package_apps_dir = extract_dir / "apps" + if not package_apps_dir.is_dir(): + raise DeployError(f"Package {ref} does not contain apps/") + + for entry in sorted(package_apps_dir.iterdir()): + target = apps_dir / entry.name + if target.exists(): + raise DeployError(f"App conflicts with an existing app: {entry.name}") + if entry.is_dir(): + shutil.copytree(entry, target) + else: + shutil.copy2(entry, target) + + return release_dir + + +def render_app_configs(release_dir, app, values): + template_dir = release_dir / "apps" / app / "config" + if not template_dir.is_dir(): + return {} + rendered = {} + for template in sorted(template_dir.glob("*.template.*")): + filename = template.name.replace(".template.", ".", 1) + rendered[filename] = render_template(template.read_text(), values) + return rendered + + +def resolve_app_envs(config, work_dir, release_dir, age_key_file): + pull_dir = work_dir / "envs" + rendered_configs = {} + for app, app_config in config["apps"].items(): + paths = [ + download_ref(ref, pull_dir / app / str(index)) + for index, ref in enumerate(app_config["env_refs"], start=1) + ] + check_env_collisions(paths) + + plaintext = "".join(decrypt_env(path, age_key_file) for path in paths) + app_env_path = release_dir / "apps" / app / ".env" + app_env_path.write_text(plaintext) + app_env_path.chmod(0o600) + + rendered_configs[app] = render_app_configs(release_dir, app, parse_dotenv(plaintext)) + + return rendered_configs + + +def list_required_networks(release_dir): + manifest = yaml.safe_load((release_dir / "apps" / "networks.yml").read_text()) + return [ + name + for name, definition in (manifest.get("networks") or {}).items() + if isinstance(definition, dict) and definition.get("external") + ] + + +def is_watchtower_managed(compose_path): + return WATCHTOWER_LABEL in Path(compose_path).read_text() + + +def archive_release(release_dir, work_dir): + archive_path = work_dir / "release.tar.gz" + with tarfile.open(archive_path, "w:gz") as tar: + for entry in sorted(release_dir.iterdir()): + tar.add(entry, arcname=entry.name) + return archive_path + + +def expand_home(path, home): + return home + path[1:] if path.startswith("~") else path + + +def bootstrap_host(connection, base_path, networks): + apps_data = f"{base_path}/apps-data" + traefik_dir = f"{apps_data}/traefik" + connection.run( + f"mkdir -p {shlex.quote(base_path)}/releases {shlex.quote(traefik_dir)}", + hide=True, + ) + for network in networks: + connection.run(f"docker network create {shlex.quote(network)} >/dev/null 2>&1 || true", hide=True) + acme_path = f"{traefik_dir}/acme.json" + connection.run( + f"[ -f {shlex.quote(acme_path)} ] || {{ touch {shlex.quote(acme_path)} && chmod 600 {shlex.quote(acme_path)}; }}", + hide=True, + ) + + +def push_release(connection, archive_path, release_path): + connection.run(f"mkdir -p {shlex.quote(release_path)}", hide=True) + connection.put(str(archive_path), remote=f"{release_path}.tar.gz") + connection.run(f"tar -xzf {shlex.quote(release_path)}.tar.gz -C {shlex.quote(release_path)}", hide=True) + connection.run(f"rm -f {shlex.quote(release_path)}.tar.gz", hide=True) + connection.run(f"chmod 600 {shlex.quote(release_path)}/apps/*/.env", hide=True) + + +def push_app_configs(connection, base_path, rendered_configs): + for app, files in rendered_configs.items(): + if not files: + continue + config_dir = f"{base_path}/apps-data/{app}/config" + connection.run(f"mkdir -p {shlex.quote(config_dir)}", hide=True) + for filename, text in files.items(): + connection.put(io.StringIO(text), remote=f"{config_dir}/{filename}") + + +def prune_releases(connection, releases_path, keep_releases): + result = connection.run(f"ls -1dt {shlex.quote(releases_path)}/*/ 2>/dev/null || true", hide=True) + releases = [line.strip().rstrip("/") for line in result.stdout.splitlines() if line.strip()] + stale = releases[keep_releases:] + if stale: + connection.run("rm -rf " + " ".join(shlex.quote(release) for release in stale), hide=True) + + +def deploy_to_host(host, archive_path, rendered_configs, all_apps, run_apps, networks, config): + connection = Connection(host) + connection.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + home = connection.run("echo $HOME", hide=True).stdout.strip() + base_path = expand_home(config.get("path", "~/flightdeck"), home) + keep_releases = config.get("keep_releases", 5) + + releases_path = f"{base_path}/releases" + current_path = f"{base_path}/current" + release_name = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + release_path = f"{releases_path}/{release_name}" + + bootstrap_host(connection, base_path, networks) + push_release(connection, archive_path, release_path) + push_app_configs(connection, base_path, rendered_configs) + for app in all_apps: + connection.run(f"mkdir -p {shlex.quote(f'{base_path}/apps-data/{app}')}", hide=True) + + connection.run(f"ln -sfn {shlex.quote(release_path)} {shlex.quote(current_path)}", hide=True) + + for app in run_apps: + compose_dir = f"{current_path}/apps/{app}" + connection.run(f"cd {shlex.quote(compose_dir)} && docker compose pull && docker compose up -d --remove-orphans") + + prune_releases(connection, releases_path, keep_releases) + connection.run("docker container prune -f && docker image prune -a -f") + + +def validate_config(config): + if not config.get("hosts"): + raise DeployError("Config must set hosts to a non-empty list") + if not config.get("app_refs"): + raise DeployError("Config must set app_refs to a non-empty list") + if not config.get("apps"): + raise DeployError("Config must set apps to a non-empty object") + if not config.get("sops_age_key"): + raise DeployError("Config must set sops_age_key") + + +def main(): + config = json.load(sys.stdin) + validate_config(config) + with tempfile.TemporaryDirectory(prefix="flightdeck-deploy-") as raw_dir: + work_dir = Path(raw_dir) + + age_key_file = work_dir / "age-key.txt" + age_key_file.write_text(config["sops_age_key"]) + age_key_file.chmod(0o600) + + release_dir = build_release(config, work_dir) + rendered_configs = resolve_app_envs(config, work_dir, release_dir, age_key_file) + archive_path = archive_release(release_dir, work_dir) + networks = list_required_networks(release_dir) + + all_apps = list(config["apps"]) + run_apps = [ + app for app in all_apps if not is_watchtower_managed(release_dir / "apps" / app / "docker-compose.yml") + ] + + for host in config["hosts"]: + print(f"Deploying to {host}") + deploy_to_host(host, archive_path, rendered_configs, all_apps, run_apps, networks, config) + + +if __name__ == "__main__": + main() diff --git a/deploy/render.py b/deploy/render.py new file mode 100644 index 0000000..476284f --- /dev/null +++ b/deploy/render.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Pure-Python equivalent of `envsubst` for rendering config templates on +the runner (values are already known there post-decryption; no need to +push plaintext to the host just to shell out to envsubst there). + +Matches real envsubst's actual behavior: only `$VAR`/`${VAR}` shell-identifier +references are substituted, missing variables become an empty string, and +anything that isn't a valid identifier reference (including bash-only +`${VAR:-default}` fallback syntax) is left untouched - envsubst doesn't +support that syntax either. +""" +import re + +VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)") + + +def render_template(text, values): + def substitute(match): + name = match.group(1) or match.group(2) + return values.get(name, "") + + return VAR_RE.sub(substitute, text) diff --git a/deploy/requirements.txt b/deploy/requirements.txt new file mode 100644 index 0000000..f459437 --- /dev/null +++ b/deploy/requirements.txt @@ -0,0 +1,2 @@ +fabric +PyYAML==6.0.2 diff --git a/deploy/resolve.py b/deploy/resolve.py new file mode 100644 index 0000000..b426b37 --- /dev/null +++ b/deploy/resolve.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Resolve and download owner/repo@tag[:asset] release refs via the gh CLI. + +Single shared implementation for every ref this deploy tool downloads +(the machinery bundle, app bundles, and per-app encrypted env assets) - +the thing three-going-on-four copies of this same ~30-line parse/resolve/ +download sequence used to be, once per caller, in ansible/deploy.yml. +""" +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path + +REF_RE = re.compile(r"^(?P[^@]+)@(?P[^:]+)(?::(?P.+))?$") + + +class RefError(Exception): + pass + + +@dataclass(frozen=True) +class ResolvedRef: + repo: str + tag: str + asset: str + + +def parse_ref(ref, default_asset=None): + match = REF_RE.fullmatch(ref) + if not match: + raise RefError(f"Invalid release ref: {ref}. Expected owner/repo@tag or owner/repo@tag:asset") + asset = match.group("asset") or default_asset + if not asset: + raise RefError(f"Invalid release ref: {ref} has no asset and no default was provided") + return ResolvedRef(repo=match.group("repo"), tag=match.group("tag"), asset=asset) + + +def resolve_latest(repo, run=subprocess.run): + result = run( + ["gh", "release", "view", "--repo", repo, "--json", "tagName", "--jq", ".tagName"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RefError(f"Could not resolve GitHub latest release for {repo}: {result.stderr.strip()}") + tag = result.stdout.strip() + if not tag or tag == "null": + raise RefError(f"Could not resolve GitHub latest release for {repo}") + return tag + + +def download_ref(ref, out_dir, default_asset=None, run=subprocess.run): + resolved = parse_ref(ref, default_asset) + tag = resolved.tag + if tag == "latest": + tag = resolve_latest(resolved.repo, run=run) + + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + result = run( + [ + "gh", + "release", + "download", + tag, + "--repo", + resolved.repo, + "--pattern", + resolved.asset, + "--dir", + str(out_dir), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RefError(f"Failed to download {resolved.asset} from {resolved.repo}@{tag}: {result.stderr.strip()}") + + path = out_dir / resolved.asset + if not path.is_file(): + raise RefError(f"{resolved.asset} was not found in {resolved.repo}@{tag}") + return path diff --git a/deploy/tests/test_collisions.py b/deploy/tests/test_collisions.py new file mode 100644 index 0000000..aa3575c --- /dev/null +++ b/deploy/tests/test_collisions.py @@ -0,0 +1,66 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "collisions.py" +SPEC = importlib.util.spec_from_file_location("collisions", MODULE_PATH) +collisions = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(collisions) + + +TRAEFIK_ENV = """\ +HTTP_PORT=ENC[AES256_GCM,data:Ab==,iv:xx==,tag:yy==,type:str] +HTTPS_PORT=ENC[AES256_GCM,data:Cd==,iv:xx==,tag:yy==,type:str] +APPS_DOMAIN=ENC[AES256_GCM,data:Ef==,iv:xx==,tag:yy==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE----- +sops_lastmodified=2026-08-19T00:00:00Z +sops_mac=ENC[AES256_GCM,data:Gh==,iv:xx==,tag:yy==,type:str] +sops_version=3.13.1 +""" + +RYBBIT_ENV = """\ +APPS_DATABASE_PASSWORD=ENC[AES256_GCM,data:Ij==,iv:xx==,tag:yy==,type:str] +APPS_KEY_HEX_32=ENC[AES256_GCM,data:Kl==,iv:xx==,tag:yy==,type:str] +sops_lastmodified=2026-08-19T00:00:00Z +sops_version=3.13.1 +""" + +COLLIDING_ENV = """\ +APPS_DOMAIN=ENC[AES256_GCM,data:Mn==,iv:xx==,tag:yy==,type:str] +sops_version=3.13.1 +""" + + +def write(directory, name, content): + path = Path(directory) / name + path.write_text(content) + return path + + +class CheckEnvCollisionsTest(unittest.TestCase): + def test_no_collision_across_clean_files(self): + with tempfile.TemporaryDirectory() as directory: + traefik = write(directory, "traefik.sops.env", TRAEFIK_ENV) + rybbit = write(directory, "rybbit.sops.env", RYBBIT_ENV) + collisions.check_env_collisions([traefik, rybbit]) # does not raise + + def test_raises_on_real_collision(self): + with tempfile.TemporaryDirectory() as directory: + traefik = write(directory, "traefik.sops.env", TRAEFIK_ENV) + colliding = write(directory, "colliding.sops.env", COLLIDING_ENV) + with self.assertRaises(collisions.CollisionError): + collisions.check_env_collisions([traefik, colliding]) + + def test_ignores_sops_metadata_keys(self): + with tempfile.TemporaryDirectory() as directory: + traefik = write(directory, "traefik.sops.env", TRAEFIK_ENV) + rybbit = write(directory, "rybbit.sops.env", RYBBIT_ENV) + keys = collisions.extract_keys(traefik) | collisions.extract_keys(rybbit) + self.assertFalse(any(key.startswith("sops_") for key in keys)) + self.assertIn("HTTP_PORT", keys) + self.assertIn("APPS_KEY_HEX_32", keys) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/tests/test_deploy.py b/deploy/tests/test_deploy.py new file mode 100644 index 0000000..e12a78f --- /dev/null +++ b/deploy/tests/test_deploy.py @@ -0,0 +1,372 @@ +import importlib.util +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from zipfile import ZipFile + +DEPLOY_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(DEPLOY_DIR)) + +MODULE_PATH = DEPLOY_DIR / "deploy.py" +SPEC = importlib.util.spec_from_file_location("deploy_entrypoint", MODULE_PATH) +deploy = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(deploy) + +# deploy.py's own `from collisions import ...` performs a real sys.path +# import, registering the canonical module here - reuse it so exception +# identity matches what deploy.py actually raises. +collisions = sys.modules["collisions"] + +NETWORKS_YML = """\ +networks: + internal: + databases: + external: true + mcp: + external: true + traefik: + external: true +""" + + +def make_zip(path, files): + path.parent.mkdir(parents=True, exist_ok=True) + with ZipFile(path, "w") as archive: + for name, content in files.items(): + archive.writestr(name, content) + return path + + +class ValidateConfigTest(unittest.TestCase): + VALID = { + "hosts": ["user@host"], + "app_refs": ["owner/repo@latest"], + "apps": {"traefik": {"env_refs": ["owner/repo@latest:a.sops.env"]}}, + "sops_age_key": "AGE-SECRET-KEY-1...", + } + + def test_accepts_valid_config(self): + deploy.validate_config(self.VALID) # does not raise + + def test_rejects_missing_hosts(self): + with self.assertRaises(deploy.DeployError): + deploy.validate_config({**self.VALID, "hosts": []}) + + def test_rejects_missing_app_refs(self): + with self.assertRaises(deploy.DeployError): + deploy.validate_config({**self.VALID, "app_refs": []}) + + def test_rejects_missing_apps(self): + with self.assertRaises(deploy.DeployError): + deploy.validate_config({**self.VALID, "apps": {}}) + + def test_rejects_missing_sops_age_key(self): + with self.assertRaises(deploy.DeployError): + deploy.validate_config({**self.VALID, "sops_age_key": ""}) + + +class ExpandHomeTest(unittest.TestCase): + def test_expands_tilde_prefix(self): + self.assertEqual(deploy.expand_home("~/flightdeck", "/home/deploy"), "/home/deploy/flightdeck") + + def test_leaves_absolute_path_untouched(self): + self.assertEqual(deploy.expand_home("/opt/flightdeck", "/home/deploy"), "/opt/flightdeck") + + +class BuildReleaseTest(unittest.TestCase): + def test_merges_app_dirs_and_shared_top_level_files(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + apps_zip = make_zip( + work_dir / "src" / "flightdeck-apps.zip", + { + "apps/common.yml": "services: {}\n", + "apps/networks.yml": NETWORKS_YML, + "apps/traefik/docker-compose.yml": "traefik: {}\n", + "apps/rybbit/docker-compose.yml": "rybbit: {}\n", + }, + ) + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return apps_zip + + config = {"app_refs": ["owner/repo@latest:apps.zip"]} + with patch.object(deploy, "download_ref", side_effect=fake_download_ref): + release_dir = deploy.build_release(config, work_dir / "work") + + self.assertTrue((release_dir / "apps" / "common.yml").is_file()) + self.assertTrue((release_dir / "apps" / "networks.yml").is_file()) + self.assertTrue((release_dir / "apps" / "traefik" / "docker-compose.yml").is_file()) + self.assertTrue((release_dir / "apps" / "rybbit" / "docker-compose.yml").is_file()) + + def test_raises_on_app_dir_conflict_across_bundles(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + first_zip = make_zip(work_dir / "src" / "a.zip", {"apps/traefik/docker-compose.yml": "a: {}\n"}) + second_zip = make_zip(work_dir / "src" / "b.zip", {"apps/traefik/docker-compose.yml": "b: {}\n"}) + zips = {"a.zip": first_zip, "b.zip": second_zip} + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return zips["a.zip"] if "a.zip" in ref else zips["b.zip"] + + config = {"app_refs": ["owner/repo@latest:a.zip", "owner/repo@latest:b.zip"]} + with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): + deploy.build_release(config, work_dir / "work") + + def test_raises_on_shared_file_conflict_across_bundles(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + first_zip = make_zip(work_dir / "src" / "a.zip", {"apps/common.yml": "a: {}\n"}) + second_zip = make_zip(work_dir / "src" / "b.zip", {"apps/common.yml": "b: {}\n"}) + zips = {"a.zip": first_zip, "b.zip": second_zip} + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return zips["a.zip"] if "a.zip" in ref else zips["b.zip"] + + config = {"app_refs": ["owner/repo@latest:a.zip", "owner/repo@latest:b.zip"]} + with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): + deploy.build_release(config, work_dir / "work") + + def test_raises_when_bundle_has_no_apps_dir(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + empty_zip = make_zip(work_dir / "src" / "empty.zip", {"README.md": "n/a\n"}) + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return empty_zip + + config = {"app_refs": ["owner/repo@latest:empty.zip"]} + with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): + deploy.build_release(config, work_dir / "work") + + +class RenderAppConfigsTest(unittest.TestCase): + def test_renders_templates_found_for_app(self): + with tempfile.TemporaryDirectory() as directory: + release_dir = Path(directory) + config_dir = release_dir / "apps" / "traefik" / "config" + config_dir.mkdir(parents=True) + (config_dir / "traefik.template.yml").write_text("email: ${APPS_ADMIN_MAIL}\n") + + rendered = deploy.render_app_configs(release_dir, "traefik", {"APPS_ADMIN_MAIL": "a@example.com"}) + + self.assertEqual(rendered, {"traefik.yml": "email: a@example.com\n"}) + + def test_returns_empty_dict_when_no_config_dir(self): + with tempfile.TemporaryDirectory() as directory: + release_dir = Path(directory) + (release_dir / "apps" / "rybbit").mkdir(parents=True) + + rendered = deploy.render_app_configs(release_dir, "rybbit", {}) + + self.assertEqual(rendered, {}) + + +class ResolveAppEnvsTest(unittest.TestCase): + def _release_dir_with_app(self, work_dir, app, template=None): + app_dir = work_dir / "release" / "apps" / app + app_dir.mkdir(parents=True) + if template is not None: + config_dir = app_dir / "config" + config_dir.mkdir() + (config_dir / f"{app}.template.yml").write_text(template) + return work_dir / "release" + + def test_writes_decrypted_env_and_renders_configs(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + release_dir = self._release_dir_with_app(work_dir, "traefik", template="email: ${APPS_ADMIN_MAIL}\n") + ciphertext = work_dir / "a.sops.env" + ciphertext.write_text("APPS_ADMIN_MAIL=ENC[...]\n") + + config = {"apps": {"traefik": {"env_refs": ["owner/repo@latest:a.sops.env"]}}} + + with ( + patch.object(deploy, "download_ref", return_value=ciphertext), + patch.object(deploy, "decrypt_env", return_value="APPS_ADMIN_MAIL=a@example.com\n"), + ): + rendered = deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") + + env_path = release_dir / "apps" / "traefik" / ".env" + self.assertEqual(env_path.read_text(), "APPS_ADMIN_MAIL=a@example.com\n") + self.assertEqual(oct(env_path.stat().st_mode)[-3:], "600") + self.assertEqual(rendered, {"traefik": {"traefik.yml": "email: a@example.com\n"}}) + + def test_raises_on_collision_within_one_apps_own_env_refs(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + release_dir = self._release_dir_with_app(work_dir, "traefik") + first_env = work_dir / "a.sops.env" + first_env.write_text("APPS_DOMAIN=ENC[AES256_GCM,data:Ab==,iv:xx==,tag:yy==,type:str]\n") + second_env = work_dir / "b.sops.env" + second_env.write_text("APPS_DOMAIN=ENC[AES256_GCM,data:Cd==,iv:xx==,tag:yy==,type:str]\n") + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return first_env if ref.endswith(":a.sops.env") else second_env + + config = { + "apps": { + "traefik": { + "env_refs": [ + "owner/repo@latest:a.sops.env", + "owner/repo@latest:b.sops.env", + ] + }, + } + } + with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(collisions.CollisionError): + deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") + + def test_allows_same_key_across_different_apps(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + release_dir = self._release_dir_with_app(work_dir, "traefik") + self._release_dir_with_app(work_dir, "rybbit") + traefik_env = work_dir / "a.sops.env" + traefik_env.write_text("APPS_DOMAIN=ENC[AES256_GCM,data:Ab==,iv:xx==,tag:yy==,type:str]\n") + rybbit_env = work_dir / "b.sops.env" + rybbit_env.write_text("APPS_DOMAIN=ENC[AES256_GCM,data:Cd==,iv:xx==,tag:yy==,type:str]\n") + + def fake_download_ref(ref, out_dir, default_asset=None, run=None): + return traefik_env if "traefik" in ref else rybbit_env + + def fake_decrypt_env(path, age_key_file, run=None): + return "APPS_DOMAIN=example.com\n" + + config = { + "apps": { + "traefik": {"env_refs": ["owner/repo@latest:hawkeye-traefik.sops.env"]}, + "rybbit": {"env_refs": ["owner/repo@latest:hawkeye-rybbit.sops.env"]}, + } + } + with ( + patch.object(deploy, "download_ref", side_effect=fake_download_ref), + patch.object(deploy, "decrypt_env", side_effect=fake_decrypt_env), + ): + deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") # does not raise + + +class ListRequiredNetworksTest(unittest.TestCase): + def test_returns_only_external_networks(self): + with tempfile.TemporaryDirectory() as directory: + release_dir = Path(directory) + apps_dir = release_dir / "apps" + apps_dir.mkdir() + (apps_dir / "networks.yml").write_text(NETWORKS_YML) + + networks = deploy.list_required_networks(release_dir) + + self.assertEqual(sorted(networks), ["databases", "mcp", "traefik"]) + self.assertNotIn("internal", networks) + + +class IsWatchtowerManagedTest(unittest.TestCase): + def test_true_when_label_present(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "docker-compose.yml" + path.write_text('labels:\n - "com.centurylinklabs.watchtower.enable=true"\n') + self.assertTrue(deploy.is_watchtower_managed(path)) + + def test_false_when_label_absent(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "docker-compose.yml" + path.write_text("services: {}\n") + self.assertFalse(deploy.is_watchtower_managed(path)) + + +class ArchiveReleaseTest(unittest.TestCase): + def test_archives_release_contents_without_wrapper_dir(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + release_dir = work_dir / "release" + (release_dir / "apps" / "traefik").mkdir(parents=True) + (release_dir / "apps" / "common.yml").write_text("services: {}\n") + (release_dir / "apps" / "traefik" / "docker-compose.yml").write_text("traefik: {}\n") + + archive_path = deploy.archive_release(release_dir, work_dir) + + with tarfile.open(archive_path) as tar: + names = set(tar.getnames()) + self.assertIn("apps/common.yml", names) + self.assertIn("apps/traefik/docker-compose.yml", names) + + +class FakeConnection: + """Stand-in for fabric.Connection - records commands/uploads instead of + opening a real SSH session, so deploy_to_host's command sequence can be + verified without a local sshd.""" + + def __init__(self, host): + self.host = host + self.client = SimpleNamespace(set_missing_host_key_policy=lambda policy: None) + self.commands = [] + self.uploads = [] + + def run(self, command, hide=False): + self.commands.append(command) + if command == "echo $HOME": + return SimpleNamespace(stdout="/home/deploy\n") + if command.startswith("ls -1dt"): + releases = "\n".join(f"/home/deploy/flightdeck/releases/rel{i}/" for i in range(7)) + return SimpleNamespace(stdout=releases + "\n") + return SimpleNamespace(stdout="") + + def put(self, local, remote): + self.uploads.append((local, remote)) + + +class DeployToHostTest(unittest.TestCase): + def test_full_sequence(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + archive_path = work_dir / "release.tar.gz" + archive_path.write_text("fake archive\n") + rendered_configs = { + "traefik": {"traefik.yml": "email: a@example.com\n"}, + "rybbit": {}, + } + config = {"hosts": ["deploy@host"], "keep_releases": 5} + + fake = FakeConnection("deploy@host") + with patch.object(deploy, "Connection", return_value=fake): + deploy.deploy_to_host( + "deploy@host", + archive_path, + rendered_configs, + all_apps=["traefik", "rybbit"], + run_apps=["rybbit"], + networks=["traefik", "databases", "mcp"], + config=config, + ) + + self.assertEqual(fake.uploads[0], (str(archive_path), fake.uploads[0][1])) + self.assertTrue(fake.uploads[0][1].endswith(".tar.gz")) + + config_upload = next(upload for upload in fake.uploads if upload[1].endswith("traefik.yml")) + self.assertEqual(config_upload[0].getvalue(), "email: a@example.com\n") + + joined = "\n".join(fake.commands) + self.assertIn("docker network create traefik", joined) + self.assertIn("docker network create databases", joined) + self.assertIn("docker network create mcp", joined) + self.assertIn("acme.json", joined) + self.assertIn("tar -xzf", joined) + self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/traefik", joined) + self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/rybbit", joined) + self.assertIn("ln -sfn", joined) + self.assertIn("apps/rybbit && docker compose pull && docker compose up -d --remove-orphans", joined) + self.assertNotIn("apps/traefik && docker compose", joined) + + prune_command = next(command for command in fake.commands if command.startswith("rm -rf") and "rel" in command) + for stale in ("rel5", "rel6"): + self.assertIn(stale, prune_command) + for kept in ("rel0", "rel1", "rel2", "rel3", "rel4"): + self.assertNotIn(kept, prune_command) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/tests/test_render.py b/deploy/tests/test_render.py new file mode 100644 index 0000000..759660c --- /dev/null +++ b/deploy/tests/test_render.py @@ -0,0 +1,39 @@ +import importlib.util +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "render.py" +SPEC = importlib.util.spec_from_file_location("render", MODULE_PATH) +render = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(render) + + +class RenderTemplateTest(unittest.TestCase): + def test_substitutes_braced_var(self): + result = render.render_template("email: ${APPS_ADMIN_MAIL}", {"APPS_ADMIN_MAIL": "a@example.com"}) + self.assertEqual(result, "email: a@example.com") + + def test_substitutes_bare_var(self): + result = render.render_template("email: $APPS_ADMIN_MAIL", {"APPS_ADMIN_MAIL": "a@example.com"}) + self.assertEqual(result, "email: a@example.com") + + def test_missing_var_becomes_empty_string(self): + result = render.render_template("email: ${APPS_ADMIN_MAIL}", {}) + self.assertEqual(result, "email: ") + + def test_leaves_bash_default_syntax_untouched(self): + text = "email: ${APPS_ADMIN_MAIL:-fallback@example.com}" + result = render.render_template(text, {"APPS_ADMIN_MAIL": "a@example.com"}) + self.assertEqual(result, text) + + def test_leaves_double_dollar_untouched(self): + result = render.render_template("price: $$5", {}) + self.assertEqual(result, "price: $$5") + + def test_substitutes_multiple_occurrences(self): + result = render.render_template("${A}-${A}-${B}", {"A": "x", "B": "y"}) + self.assertEqual(result, "x-x-y") + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/tests/test_resolve.py b/deploy/tests/test_resolve.py new file mode 100644 index 0000000..1b8c488 --- /dev/null +++ b/deploy/tests/test_resolve.py @@ -0,0 +1,133 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +MODULE_PATH = Path(__file__).resolve().parents[1] / "resolve.py" +SPEC = importlib.util.spec_from_file_location("resolve", MODULE_PATH) +resolve = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolve) + + +def fake_run(calls, results): + def run(cmd, **kwargs): + calls.append(cmd) + return results.pop(0) + + return run + + +def ok(stdout=""): + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + +def fail(stderr="boom"): + return SimpleNamespace(returncode=1, stdout="", stderr=stderr) + + +class ParseRefTest(unittest.TestCase): + def test_parses_explicit_asset(self): + resolved = resolve.parse_ref("owner/repo@v1.2.3:thing.zip") + self.assertEqual(resolved.repo, "owner/repo") + self.assertEqual(resolved.tag, "v1.2.3") + self.assertEqual(resolved.asset, "thing.zip") + + def test_applies_default_asset(self): + resolved = resolve.parse_ref("owner/repo@latest", default_asset="flightdeck.zip") + self.assertEqual(resolved.asset, "flightdeck.zip") + + def test_explicit_asset_overrides_default(self): + resolved = resolve.parse_ref("owner/repo@latest:custom.zip", default_asset="flightdeck.zip") + self.assertEqual(resolved.asset, "custom.zip") + + def test_rejects_missing_at(self): + with self.assertRaises(resolve.RefError): + resolve.parse_ref("owner/repo-v1.2.3") + + def test_rejects_missing_asset_and_default(self): + with self.assertRaises(resolve.RefError): + resolve.parse_ref("owner/repo@v1.2.3") + + def test_rejects_empty_tag(self): + with self.assertRaises(resolve.RefError): + resolve.parse_ref("owner/repo@", default_asset="a.zip") + + +class ResolveLatestTest(unittest.TestCase): + def test_resolves_tag(self): + calls = [] + run = fake_run(calls, [ok("v1.2.3\n")]) + tag = resolve.resolve_latest("owner/repo", run=run) + self.assertEqual(tag, "v1.2.3") + self.assertIn("release", calls[0]) + self.assertIn("view", calls[0]) + + def test_raises_on_null(self): + calls = [] + run = fake_run(calls, [ok("null\n")]) + with self.assertRaises(resolve.RefError): + resolve.resolve_latest("owner/repo", run=run) + + def test_raises_on_failure(self): + calls = [] + run = fake_run(calls, [fail("no releases")]) + with self.assertRaises(resolve.RefError): + resolve.resolve_latest("owner/repo", run=run) + + +class DownloadRefTest(unittest.TestCase): + def test_downloads_pinned_tag(self): + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + asset = out / "thing.zip" + + def run(cmd, **kwargs): + if "download" in cmd: + asset.write_text("data") + return ok() + + path = resolve.download_ref("owner/repo@v1.2.3:thing.zip", out, run=run) + self.assertEqual(path, asset) + self.assertTrue(path.is_file()) + + def test_resolves_latest_before_download(self): + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + asset = out / "flightdeck.zip" + calls = [] + + def run(cmd, **kwargs): + calls.append(cmd) + if "view" in cmd: + return ok("v9.9.9\n") + asset.write_text("data") + return ok() + + resolve.download_ref("owner/repo@latest", out, default_asset="flightdeck.zip", run=run) + download_call = [c for c in calls if "download" in c][0] + self.assertIn("v9.9.9", download_call) + + def test_raises_when_asset_missing_after_download(self): + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + + def run(cmd, **kwargs): + return ok() # succeeds but never writes the file + + with self.assertRaises(resolve.RefError): + resolve.download_ref("owner/repo@v1.2.3:thing.zip", out, run=run) + + def test_raises_on_download_failure(self): + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + + def run(cmd, **kwargs): + return fail("not found") + + with self.assertRaises(resolve.RefError): + resolve.download_ref("owner/repo@v1.2.3:thing.zip", out, run=run) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/tests/test_vault.py b/deploy/tests/test_vault.py new file mode 100644 index 0000000..3df2d3d --- /dev/null +++ b/deploy/tests/test_vault.py @@ -0,0 +1,69 @@ +import importlib.util +import unittest +from pathlib import Path +from types import SimpleNamespace + +MODULE_PATH = Path(__file__).resolve().parents[1] / "vault.py" +SPEC = importlib.util.spec_from_file_location("vault", MODULE_PATH) +vault = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(vault) + + +def fake_run(calls, results): + def run(cmd, **kwargs): + calls.append((cmd, kwargs)) + return results.pop(0) + + return run + + +def ok(stdout=""): + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + +def fail(stderr="boom"): + return SimpleNamespace(returncode=1, stdout="", stderr=stderr) + + +class DecryptEnvTest(unittest.TestCase): + def test_returns_decrypted_stdout(self): + calls = [] + run = fake_run(calls, [ok("APPS_DOMAIN=example.com\n")]) + plaintext = vault.decrypt_env("traefik.sops.env", "/tmp/key.txt", run=run) + self.assertEqual(plaintext, "APPS_DOMAIN=example.com\n") + + def test_passes_sops_decrypt_dotenv_args(self): + calls = [] + run = fake_run(calls, [ok()]) + vault.decrypt_env("traefik.sops.env", "/tmp/key.txt", run=run) + cmd, kwargs = calls[0] + self.assertEqual(cmd, ["sops", "decrypt", "--input-type", "dotenv", "--output-type", "dotenv", "traefik.sops.env"]) + self.assertEqual(kwargs["env"]["SOPS_AGE_KEY_FILE"], "/tmp/key.txt") + + def test_raises_on_failure(self): + calls = [] + run = fake_run(calls, [fail("no matching key found")]) + with self.assertRaises(vault.VaultError): + vault.decrypt_env("traefik.sops.env", "/tmp/key.txt", run=run) + + +class ParseDotenvTest(unittest.TestCase): + def test_parses_key_value_pairs(self): + values = vault.parse_dotenv("APPS_DOMAIN=example.com\nHTTP_PORT=80\n") + self.assertEqual(values, {"APPS_DOMAIN": "example.com", "HTTP_PORT": "80"}) + + def test_splits_only_on_first_equals(self): + values = vault.parse_dotenv("APPS_HTPASSWD=user:pass=word\n") + self.assertEqual(values["APPS_HTPASSWD"], "user:pass=word") + + def test_ignores_lines_without_equals(self): + values = vault.parse_dotenv("not a valid line\nAPPS_DOMAIN=example.com\n") + self.assertEqual(values, {"APPS_DOMAIN": "example.com"}) + + def test_ignores_blank_lines(self): + values = vault.parse_dotenv("\n\nAPPS_DOMAIN=example.com\n") + self.assertEqual(values, {"APPS_DOMAIN": "example.com"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/vault.py b/deploy/vault.py new file mode 100644 index 0000000..4209427 --- /dev/null +++ b/deploy/vault.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Decrypt SOPS-encrypted dotenv vault assets on the runner. + +Symmetric counterpart to encrypt-env/action.yml's `sops encrypt` call - +same dotenv input/output type, same tool, just decrypt instead of encrypt. +""" +import os +import subprocess + + +class VaultError(Exception): + pass + + +def decrypt_env(sops_env_path, age_key_file, run=subprocess.run): + result = run( + ["sops", "decrypt", "--input-type", "dotenv", "--output-type", "dotenv", str(sops_env_path)], + capture_output=True, + text=True, + env={**os.environ, "SOPS_AGE_KEY_FILE": str(age_key_file)}, + ) + if result.returncode != 0: + raise VaultError(f"Failed to decrypt {sops_env_path}: {result.stderr.strip()}") + return result.stdout + + +def parse_dotenv(text): + values = {} + for line in text.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + if not key: + continue + values[key] = value + return values diff --git a/down.sh b/down.sh deleted file mode 100755 index c1b0d9b..0000000 --- a/down.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -source "$(dirname "$0")/lib.sh" -set -a -source .env -set +a - -if [ $# -gt 0 ]; then - apps=("$@") -else - parse_apps "$APPS" -fi - -for app in "${apps[@]}" -do - require_app_compose "${app}" - "$(dirname "$0")/generate-env.sh" "${app}" - set -a - source "./apps/${app}/.env" - set +a - docker compose -f "./apps/${app}/docker-compose.yml" down -done diff --git a/generate-env.sh b/generate-env.sh deleted file mode 100755 index c084666..0000000 --- a/generate-env.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -source "$(dirname "$0")/lib.sh" -set -a -source .env -set +a - -if [ $# -gt 0 ]; then - apps=("$@") -else - parse_apps "$APPS" -fi - -for app in "${apps[@]}"; do - generate_env "${app}" - echo "Generated env: ${app}" -done diff --git a/lib.sh b/lib.sh deleted file mode 100644 index 4b5dc4d..0000000 --- a/lib.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -validate_app_name() { - local app - - app="$1" - if [[ ! "$app" =~ ^[a-z][a-z0-9-]*$ ]]; then - echo "Invalid app name: $app" >&2 - return 1 - fi -} - -app_env_prefix() { - local app="$1" - - validate_app_name "$app" || return 1 - echo "$app" | tr '[:lower:]-' '[:upper:]_' -} - -parse_apps() { - local raw="${1:-}" - local app - local parsed_apps - - apps=() - raw="${raw//$'\n'/,}" - - IFS=',' read -r -a parsed_apps <<< "$raw" - for app in "${parsed_apps[@]}"; do - app="${app#"${app%%[![:space:]]*}"}" - app="${app%"${app##*[![:space:]]}"}" - - if [[ -z "$app" ]]; then - echo "Invalid APPS value: empty app name in '$raw'" >&2 - return 1 - fi - validate_app_name "$app" || return 1 - - apps+=("$app") - done - - if [[ ${#apps[@]} -eq 0 ]]; then - echo "APPS must contain at least one app name" >&2 - return 1 - fi -} - -require_app_compose() { - local app="$1" - - validate_app_name "$app" || return 1 - - if [[ ! -f "./apps/${app}/docker-compose.yml" ]]; then - echo "Missing compose file for app: ./apps/${app}/docker-compose.yml" >&2 - return 1 - fi -} - -generate_env() { - local app="$1" - local prefix - local output="./apps/${app}/.env" - - require_app_compose "$app" - prefix="$(app_env_prefix "$app")" - - { - cat .env - echo "APP_NAME=${app}" - } | awk -v prefix="${prefix}" ' - /^[[:space:]]*#/ { next } - /^[[:space:]]*$/ { next } - /=/ { - key = substr($0, 1, index($0, "=") - 1) - if (key != "APP_NAME" && key !~ /^APPS_/ && index(key, prefix "_") != 1) { next } - vals[key] = $0 - if (!(key in seen)) { order[++n] = key; seen[key] = 1 } - } - END { for (i = 1; i <= n; i++) print vals[order[i]] } - ' | { echo "# Auto-generated by up.sh — do not edit. Use ./up.sh in project root to regenerate."; cat; } > "$output" -} diff --git a/logs.sh b/logs.sh deleted file mode 100755 index 876b895..0000000 --- a/logs.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -source "$(dirname "$0")/lib.sh" -set -a -source .env -set +a - -if [ $# -eq 1 ]; then - app="$1" -else - parse_apps "$APPS" - echo "Usage: $0 " - echo "Available apps: ${apps[*]}" - exit 1 -fi - -require_app_compose "${app}" -generate_env "${app}" -set -a -source "./apps/${app}/.env" -set +a -docker compose -f "./apps/${app}/docker-compose.yml" logs -f diff --git a/restart.sh b/restart.sh deleted file mode 100755 index 07e5671..0000000 --- a/restart.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -source "$(dirname "$0")/lib.sh" -set -a -source .env -set +a - -if [ $# -gt 0 ]; then - apps=("$@") -else - parse_apps "$APPS" -fi - -for app in "${apps[@]}"; do - "$(dirname "$0")/down.sh" "$app" - "$(dirname "$0")/up.sh" "$app" -done diff --git a/targets/hawkeye.yml b/targets/hawkeye.yml index 43b79c5..6a1a6dc 100644 --- a/targets/hawkeye.yml +++ b/targets/hawkeye.yml @@ -1,11 +1,12 @@ -flightdeck_ref: rubykatzen/flightdeck@latest -env_refs: - - rubykatzen/flightdeck@latest:hawkeye.sops.env app_refs: - rubykatzen/flightdeck@latest apps: - - traefik - - rybbit + traefik: + env_refs: + - rubykatzen/flightdeck@latest:hawkeye-traefik.sops.env + rybbit: + env_refs: + - rubykatzen/flightdeck@latest:hawkeye-rybbit.sops.env hosts: - rubykatzen-com@100.75.50.2 credentials: @@ -14,3 +15,4 @@ credentials: secrets: ssh_private_key: DEPLOY_SSH_PRIVATE_KEY tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET + sops_age_key: HAWKEYE_AGE_PRIVATE_KEY diff --git a/up.sh b/up.sh deleted file mode 100755 index 8f441fb..0000000 --- a/up.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set -e - -ensure_file() { - local source_file="$1" - local target_file="$2" - - if [[ ! -f "$target_file" ]]; then - cp "$source_file" "$target_file" - echo "Created: $target_file" - fi -} - -ensure_network() { - local network="$1" - - if ! docker network inspect "$network" >/dev/null 2>&1; then - docker network create "$network" >/dev/null - echo "Created Docker network: $network" - fi -} - -source "$(dirname "$0")/lib.sh" - -ensure_file .env.example .env -mkdir -p apps-data/traefik -touch apps-data/traefik/acme.json -chmod 600 apps-data/traefik/acme.json -ensure_network traefik -ensure_network databases -ensure_network mcp - -set -a -source .env -set +a - -if [ $# -gt 0 ]; then - apps=("$@") -else - parse_apps "$APPS" -fi - -for app in "${apps[@]}" -do - ( - echo "Starting: ${app}" - - require_app_compose "${app}" - "$(dirname "$0")/generate-env.sh" "${app}" - - set -a - source "./apps/${app}/.env" - set +a - - mkdir -p "./apps-data/${app}" - - config_template_dir="./apps/${app}/config" - config_dir="./apps-data/${app}/config" - - if [[ -d "$config_template_dir" ]]; then - mkdir -p "$config_dir" - for template in "$config_template_dir"/*.template.*; do - [[ -e "$template" ]] || continue - filename="$(basename "$template")" - filename="${filename/.template./.}" - envsubst < "$template" > "$config_dir/$filename" - echo "Generated: $config_dir/$filename" - done - fi - - docker compose -f "./apps/${app}/docker-compose.yml" pull - docker compose -f "./apps/${app}/docker-compose.yml" up -d --remove-orphans - ) -done - -docker container prune -f -docker image prune -a -f diff --git a/vaults/hawkeye-rybbit.yml b/vaults/hawkeye-rybbit.yml new file mode 100644 index 0000000..09e1ee4 --- /dev/null +++ b/vaults/hawkeye-rybbit.yml @@ -0,0 +1,8 @@ +asset: hawkeye-rybbit.sops.env +keys: + - hawkeye +env: + APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN + APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER + APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD + APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 diff --git a/vaults/hawkeye-traefik.yml b/vaults/hawkeye-traefik.yml new file mode 100644 index 0000000..25535ec --- /dev/null +++ b/vaults/hawkeye-traefik.yml @@ -0,0 +1,8 @@ +asset: hawkeye-traefik.sops.env +keys: + - hawkeye +env: + APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL + APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN + HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT + HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT diff --git a/vaults/hawkeye.yml b/vaults/hawkeye.yml deleted file mode 100644 index 12d7b9b..0000000 --- a/vaults/hawkeye.yml +++ /dev/null @@ -1,13 +0,0 @@ -asset: hawkeye.sops.env -keys: - - hawkeye -env: - APPS_DOMAIN: RUBYKATZEN_COM_DOMAIN - APPS_ADMIN_MAIL: RUBYKATZEN_COM_ADMIN_MAIL - APPS_CERTIFICATE_RESOLVER: RUBYKATZEN_COM_CERT_RESOLVER - APPS_CLOUDFLARE_DNS_API_TOKEN: RUBYKATZEN_COM_CLOUDFLARE_TOKEN - APPS_DATABASE_PASSWORD: RUBYKATZEN_COM_DATABASE_PASSWORD - APPS_KEY_HEX_32: RUBYKATZEN_COM_KEY_HEX_32 - APPS_TIMEZONE: RUBYKATZEN_COM_TIMEZONE - TRAEFIK_HTTP_PORT: RUBYKATZEN_COM_TRAEFIK_HTTP_PORT - TRAEFIK_HTTPS_PORT: RUBYKATZEN_COM_TRAEFIK_HTTPS_PORT