diff --git a/.github/actions/deploy/README.md b/.github/actions/deploy/README.md new file mode 100644 index 0000000..2e7350f --- /dev/null +++ b/.github/actions/deploy/README.md @@ -0,0 +1,36 @@ +# deploy + +Composite GitHub Action that runs a push-based deploy against a target manifest already present in the caller's own checkout - resolves and downloads every release ref, decrypts and renders each app's env and config, merges the release, and pushes the finished result to each host over SSH. See the main [README](../../../README.md#automated-deploy) for the full sequence. + +## Usage + +```yaml +jobs: + deploy: + strategy: + matrix: ${{ fromJson(needs.load-targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: rubykatzen/flightdeck/.github/actions/deploy@main + with: + target-manifest: ${{ matrix.manifest }} # required, path in this repository + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + sops-age-key: ${{ secrets[matrix.sops_age_key_secret] }} + github-token: ${{ secrets.GITHUB_TOKEN }} + # tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) + # tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # required only if tailscale-oauth-client-id is set + # tailscale-tags: tag:ci # default: tag:ci + - name: Notify Telegram + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Deploy: ${{ matrix.name }} updated" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} +``` + +Unlike a `workflow_call` reusable workflow, this action doesn't check out anything itself - it reads `target-manifest` from whatever the caller's own preceding `actions/checkout` step already put on disk, and its own code (`deploy.py` and friends) comes along automatically via `$GITHUB_ACTION_PATH` whenever it's referenced as `owner/repo/.github/actions/deploy@ref`. This is also why there's no `target-manifest-ref` input here: if the caller needs a specific ref (e.g. a just-published release tag), it just checks out that ref itself before this step runs, the same way every other job in this repository already does. + +`sops-age-key`/`ssh-private-key` are secret *values*, resolved by the caller from the target manifest's own `sops_age_key_secret`/`ssh_private_key_secret` fields (GitHub Secret *names* - see the main README's "Vaults And Targets" section) - this action never reads the manifest's credential fields itself, since it never has access to `secrets.*` by name. + +This action has no notification logic of its own, same as [`renovate`](../renovate) - notifying is the caller's job, as a plain following step. Unlike `renovate`, there's no "did anything change" output to gate it on: reaching that step at all already means the `deploy` step above it succeeded, so it just runs unconditionally. diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml new file mode 100644 index 0000000..760c524 --- /dev/null +++ b/.github/actions/deploy/action.yml @@ -0,0 +1,65 @@ +name: Deploy +description: Push-based deploy against a target manifest already present in the caller's own checkout. +inputs: + target-manifest: + description: Path to the target manifest file, e.g. targets/heimdall.yml - already checked out by the caller. + required: true + ssh-private-key: + description: SSH private key used to connect to the hosts. + required: true + sops-age-key: + description: Private SOPS age key used to decrypt this target's vault-sourced env, on the runner. + required: true + github-token: + description: Token used to resolve/download release refs (app bundles, encrypted env sources). + required: true + tailscale-oauth-client-id: + description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). + required: false + default: "" + tailscale-oauth-secret: + description: Tailscale OAuth client secret used to join the tailnet. Required only when tailscale-oauth-client-id is set. + required: false + default: "" + tailscale-tags: + description: Comma-separated ACL tags for the ephemeral tailnet node. + required: false + default: tag:ci +runs: + using: composite + steps: + - name: Install deploy dependencies + shell: bash + run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" + - name: Install sops + shell: bash + env: + SOPS_VERSION: "3.13.1" + run: | + curl --fail --location --silent --show-error \ + "https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64" \ + --output /usr/local/bin/sops + chmod +x /usr/local/bin/sops + - uses: tailscale/github-action@v4 + if: inputs.tailscale-oauth-client-id != '' + with: + oauth-client-id: ${{ inputs.tailscale-oauth-client-id }} + oauth-secret: ${{ inputs.tailscale-oauth-secret }} + tags: ${{ inputs.tailscale-tags }} + - name: Load SSH key + shell: bash + run: | + eval "$(ssh-agent -s)" + echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" + echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" + ssh-add - <<< "${{ inputs.ssh-private-key }}" + - name: Run deploy + shell: bash + env: + TARGET_MANIFEST: ${{ inputs.target-manifest }} + SOPS_AGE_KEY: ${{ inputs.sops-age-key }} + GH_TOKEN: ${{ inputs.github-token }} + run: | + jq -n --arg target_manifest "$TARGET_MANIFEST" --arg sops_age_key "$SOPS_AGE_KEY" \ + '{target_manifest: $target_manifest, sops_age_key: $sops_age_key}' \ + | python3 "$GITHUB_ACTION_PATH/deploy.py" diff --git a/deploy/collisions.py b/.github/actions/deploy/collisions.py similarity index 100% rename from deploy/collisions.py rename to .github/actions/deploy/collisions.py diff --git a/deploy/deploy.py b/.github/actions/deploy/deploy.py similarity index 92% rename from deploy/deploy.py rename to .github/actions/deploy/deploy.py index 4e62c8f..3f7024d 100644 --- a/deploy/deploy.py +++ b/.github/actions/deploy/deploy.py @@ -6,8 +6,13 @@ nothing but Docker and Docker Compose: no sops, no age key, no gh, no flightdeck scripts of any kind. -Reads a JSON config from stdin (see README's "deploy-shared.yml" section -for the exact shape). +Reads a JSON config from stdin: {"target_manifest": "targets/heimdall.yml", +"sops_age_key": "..."}. `target_manifest` is a path to that target's own +manifest, parsed here rather than flattened into separate fields by the +caller - the same shape encrypt-env's render-env.py already uses for +vault manifests. `sops_age_key` can't come from that file, since it's a +secret value, not YAML-safe config - the calling `deploy` action passes +it separately (see README's "deploy" section for the exact contract). """ import json import shlex @@ -192,6 +197,10 @@ def deploy_to_host(host, archive_path, apps, networks, config, release_name): connection.run("docker container prune -f && docker image prune -a -f") +def load_target(path): + return yaml.safe_load(Path(path).read_text()) + + def validate_config(config): if not config.get("hosts"): raise DeployError("Config must set hosts to a non-empty list") @@ -204,7 +213,9 @@ def validate_config(config): def main(): - config = json.load(sys.stdin) + stdin_config = json.load(sys.stdin) + config = load_target(stdin_config["target_manifest"]) + config["sops_age_key"] = stdin_config.get("sops_age_key") validate_config(config) with tempfile.TemporaryDirectory(prefix="flightdeck-deploy-") as raw_dir: work_dir = Path(raw_dir) diff --git a/deploy/render.py b/.github/actions/deploy/render.py similarity index 100% rename from deploy/render.py rename to .github/actions/deploy/render.py diff --git a/deploy/requirements.txt b/.github/actions/deploy/requirements.txt similarity index 100% rename from deploy/requirements.txt rename to .github/actions/deploy/requirements.txt diff --git a/deploy/resolve.py b/.github/actions/deploy/resolve.py similarity index 100% rename from deploy/resolve.py rename to .github/actions/deploy/resolve.py diff --git a/deploy/tests/test_collisions.py b/.github/actions/deploy/tests/test_collisions.py similarity index 100% rename from deploy/tests/test_collisions.py rename to .github/actions/deploy/tests/test_collisions.py diff --git a/deploy/tests/test_deploy.py b/.github/actions/deploy/tests/test_deploy.py similarity index 85% rename from deploy/tests/test_deploy.py rename to .github/actions/deploy/tests/test_deploy.py index c129edc..5cfe501 100644 --- a/deploy/tests/test_deploy.py +++ b/.github/actions/deploy/tests/test_deploy.py @@ -1,4 +1,5 @@ import importlib.util +import io import json import sys import tarfile @@ -42,6 +43,22 @@ def make_zip(path, files): return path +class LoadTargetTest(unittest.TestCase): + def test_parses_a_target_manifest(self): + with tempfile.TemporaryDirectory() as directory: + manifest_path = Path(directory) / "heimdall.yml" + manifest_path.write_text( + "hosts: [user@host]\napp_refs: [owner/repo@latest]\n" + "apps:\n traefik:\n env_refs: [owner/repo@latest:a.sops.env]\n" + ) + + target = deploy.load_target(manifest_path) + + self.assertEqual(target["hosts"], ["user@host"]) + self.assertEqual(target["app_refs"], ["owner/repo@latest"]) + self.assertEqual(target["apps"]["traefik"]["env_refs"], ["owner/repo@latest:a.sops.env"]) + + class ValidateConfigTest(unittest.TestCase): VALID = { "hosts": ["user@host"], @@ -456,5 +473,57 @@ def test_no_previous_manifest_stops_nothing(self): self.assertNotIn("docker compose down", joined) +class MainTest(unittest.TestCase): + def test_reads_target_manifest_and_merges_sops_age_key(self): + with tempfile.TemporaryDirectory() as directory: + manifest_path = Path(directory) / "heimdall.yml" + manifest_path.write_text( + "hosts: [deploy@host]\n" + "app_refs: [owner/repo@latest]\n" + "apps:\n traefik:\n env_refs: [owner/repo@latest:a.sops.env]\n" + ) + stdin_config = {"target_manifest": str(manifest_path), "sops_age_key": "AGE-SECRET-KEY-1..."} + + with ( + patch.object(sys, "stdin", io.StringIO(json.dumps(stdin_config))), + patch.object( + deploy, "build_release", return_value=(Path(directory) / "release", ["owner/repo@v1.0.0"]) + ) as fake_build_release, + patch.object( + deploy, + "resolve_app_envs", + return_value={"traefik": ["owner/repo@v1.0.0:a.sops.env"]}, + ) as fake_resolve_app_envs, + patch.object(deploy, "write_release_manifest"), + patch.object(deploy, "archive_release", return_value=Path(directory) / "release.tar.gz"), + patch.object(deploy, "list_required_networks", return_value=[]), + patch.object(deploy, "deploy_to_host") as fake_deploy_to_host, + ): + deploy.main() + + config_arg = fake_build_release.call_args[0][0] + self.assertEqual(config_arg["hosts"], ["deploy@host"]) + self.assertEqual(config_arg["app_refs"], ["owner/repo@latest"]) + self.assertEqual(config_arg["sops_age_key"], "AGE-SECRET-KEY-1...") + + fake_resolve_app_envs.assert_called_once() + fake_deploy_to_host.assert_called_once() + host, archive_path, apps, networks, deploy_config = fake_deploy_to_host.call_args[0][:5] + expected = ("deploy@host", Path(directory) / "release.tar.gz", ["traefik"], [], config_arg) + self.assertEqual((host, archive_path, apps, networks, deploy_config), expected) + + def test_raises_when_sops_age_key_missing(self): + with tempfile.TemporaryDirectory() as directory: + manifest_path = Path(directory) / "heimdall.yml" + manifest_path.write_text("hosts: [deploy@host]\napp_refs: [owner/repo@latest]\napps:\n traefik: {}\n") + stdin_config = {"target_manifest": str(manifest_path), "sops_age_key": ""} + + with ( + patch.object(sys, "stdin", io.StringIO(json.dumps(stdin_config))), + self.assertRaises(deploy.DeployError), + ): + deploy.main() + + if __name__ == "__main__": unittest.main() diff --git a/deploy/tests/test_render.py b/.github/actions/deploy/tests/test_render.py similarity index 100% rename from deploy/tests/test_render.py rename to .github/actions/deploy/tests/test_render.py diff --git a/deploy/tests/test_resolve.py b/.github/actions/deploy/tests/test_resolve.py similarity index 100% rename from deploy/tests/test_resolve.py rename to .github/actions/deploy/tests/test_resolve.py diff --git a/deploy/tests/test_vault.py b/.github/actions/deploy/tests/test_vault.py similarity index 100% rename from deploy/tests/test_vault.py rename to .github/actions/deploy/tests/test_vault.py diff --git a/deploy/vault.py b/.github/actions/deploy/vault.py similarity index 100% rename from deploy/vault.py rename to .github/actions/deploy/vault.py diff --git a/.github/actions/load-targets-matrix/README.md b/.github/actions/load-targets-matrix/README.md new file mode 100644 index 0000000..5ebd5cc --- /dev/null +++ b/.github/actions/load-targets-matrix/README.md @@ -0,0 +1,18 @@ +# load-targets-matrix + +Composite GitHub Action that reads every target manifest in a directory into a GitHub Actions matrix, validating each one's required shape. Sibling to [`load-vaults-matrix`](../load-vaults-matrix), same shape, for `vaults/*.yml` instead. + +## Usage + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/load-targets-matrix@main + id: load-targets + with: + # directory: targets # optional, default shown +``` + +The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Each matrix item merges the manifest's own top-level YAML fields with `name` (the file's basename) and `manifest` (its path). + +Every manifest must set `hosts` (non-empty list), `app_refs` (non-empty list), `apps` (non-empty object), `ssh_private_key_secret` (non-empty string - the name of the GitHub Secret holding this target's SSH private key), and `sops_age_key_secret` (non-empty string - the name of the GitHub Secret holding this target's private SOPS age key). This fails the matrix build immediately, with an error naming the specific broken target, rather than letting a malformed manifest reach the [`deploy`](../deploy)/[`renovate`](../renovate) actions much later, after a checkout and dependency install on a different job entirely. + +Files may use either the `.yml` or `.yaml` extension. Filenames must match `^[a-z0-9][a-z0-9-]*$` and be unique per directory; duplicate top-level YAML keys within a manifest are rejected. diff --git a/.github/actions/load-targets-matrix/action.yml b/.github/actions/load-targets-matrix/action.yml new file mode 100644 index 0000000..27072d0 --- /dev/null +++ b/.github/actions/load-targets-matrix/action.yml @@ -0,0 +1,27 @@ +name: Load targets matrix +description: Read every target manifest in a directory into a GitHub Actions matrix, validating each one's required shape. +inputs: + directory: + description: Directory containing target manifests. + required: false + default: targets +outputs: + matrix: + description: JSON strategy matrix, one item per target manifest, merging its parsed YAML fields with name and manifest. + value: ${{ steps.load.outputs.matrix }} + count: + description: Number of targets in the matrix. + value: ${{ steps.load.outputs.count }} +runs: + using: composite + steps: + - name: Install dependencies + shell: bash + run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" + - name: Load matrix + id: load + shell: bash + env: + DIRECTORY: ${{ inputs.directory }} + run: | + python3 "$GITHUB_ACTION_PATH/scripts/load-targets-matrix.py" --directory "$DIRECTORY" diff --git a/.github/actions/load-yaml-matrix/requirements.txt b/.github/actions/load-targets-matrix/requirements.txt similarity index 100% rename from .github/actions/load-yaml-matrix/requirements.txt rename to .github/actions/load-targets-matrix/requirements.txt diff --git a/.github/actions/load-targets-matrix/scripts/load-targets-matrix.py b/.github/actions/load-targets-matrix/scripts/load-targets-matrix.py new file mode 100644 index 0000000..bbded8d --- /dev/null +++ b/.github/actions/load-targets-matrix/scripts/load-targets-matrix.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import sys +from pathlib import Path + +import yaml + +NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") + + +class ManifestError(Exception): + pass + + +class UniqueKeyLoader(yaml.SafeLoader): + pass + + +def construct_mapping(loader, node, deep=False): + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise ManifestError(f"duplicate YAML key: {key}") + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping, +) + + +def load_manifest(path): + try: + value = yaml.load(path.read_text(), Loader=UniqueKeyLoader) + except yaml.YAMLError as exc: + raise ManifestError(f"{path} is not valid YAML: {exc}") from exc + if not isinstance(value, dict): + raise ManifestError(f"{path} must contain a YAML mapping") + return value + + +def validate_target(name, manifest): + if not isinstance(manifest.get("hosts"), list) or not manifest["hosts"]: + raise ManifestError(f"target {name!r} must set hosts to a non-empty list") + if not isinstance(manifest.get("app_refs"), list) or not manifest["app_refs"]: + raise ManifestError(f"target {name!r} must set app_refs to a non-empty list") + if not isinstance(manifest.get("apps"), dict) or not manifest["apps"]: + raise ManifestError(f"target {name!r} must set apps to a non-empty object") + if not isinstance(manifest.get("ssh_private_key_secret"), str) or not manifest["ssh_private_key_secret"]: + raise ManifestError(f"target {name!r} must set ssh_private_key_secret to a non-empty string") + if not isinstance(manifest.get("sops_age_key_secret"), str) or not manifest["sops_age_key_secret"]: + raise ManifestError(f"target {name!r} must set sops_age_key_secret to a non-empty string") + + +def build_matrix(directory): + paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) + if not paths: + raise ManifestError(f"no manifests found in {directory}") + include = [] + seen_names = set() + for path in paths: + name = path.stem + if not NAME_RE.fullmatch(name): + raise ManifestError(f"invalid manifest filename: {path.name}") + if name in seen_names: + raise ManifestError(f"duplicate manifest name: {name}") + seen_names.add(name) + manifest = load_manifest(path) + validate_target(name, manifest) + item = {"name": name, "manifest": str(path)} + item.update(manifest) + include.append(item) + return {"include": include} + + +def write_github_output(name, value): + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + output.write(f"{name}={value}\n") + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--directory", required=True, type=Path) + args = parser.parse_args(argv) + try: + matrix = build_matrix(args.directory) + encoded = json.dumps(matrix, separators=(",", ":")) + write_github_output("matrix", encoded) + write_github_output("count", len(matrix["include"])) + except ManifestError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/load-targets-matrix/tests/test_load_targets_matrix.py b/.github/actions/load-targets-matrix/tests/test_load_targets_matrix.py new file mode 100644 index 0000000..90c91c6 --- /dev/null +++ b/.github/actions/load-targets-matrix/tests/test_load_targets_matrix.py @@ -0,0 +1,114 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-targets-matrix.py" +SPEC = importlib.util.spec_from_file_location("load_targets_matrix", MODULE_PATH) +load_targets_matrix = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(load_targets_matrix) + + +HEIMDALL = """\ +app_refs: [rubykatzen/flightdeck@latest] +apps: + traefik: {} +hosts: [root@100.75.53.13] +ssh_private_key_secret: DEPLOY_SSH_PRIVATE_KEY +sops_age_key_secret: HEIMDALL_AGE_PRIVATE_KEY +""" + +MAINFRAME = """\ +app_refs: [rubykatzen/flightdeck@latest, owner/extra-apps@latest] +apps: + rybbit: {} +hosts: [deploy@app1.example.com, deploy@app2.example.com] +path: ~/flightdeck +ssh_private_key_secret: DEPLOY_SSH_PRIVATE_KEY +sops_age_key_secret: MAINFRAME_AGE_PRIVATE_KEY +""" + + +class LoadTargetsMatrixTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.directory = Path(self.temporary_directory.name) + (self.directory / "heimdall.yml").write_text(HEIMDALL) + (self.directory / "mainframe.yml").write_text(MAINFRAME) + + def test_builds_matrix_from_all_manifests(self): + matrix = load_targets_matrix.build_matrix(self.directory) + self.assertEqual( + sorted(item["name"] for item in matrix["include"]), + ["heimdall", "mainframe"], + ) + + def test_merges_manifest_fields_with_name_and_manifest(self): + matrix = load_targets_matrix.build_matrix(self.directory) + item = next(item for item in matrix["include"] if item["name"] == "heimdall") + self.assertEqual(item["manifest"], str(self.directory / "heimdall.yml")) + self.assertEqual(item["hosts"], ["root@100.75.53.13"]) + self.assertEqual(item["ssh_private_key_secret"], "DEPLOY_SSH_PRIVATE_KEY") + self.assertEqual(item["sops_age_key_secret"], "HEIMDALL_AGE_PRIVATE_KEY") + + def test_rejects_empty_directory(self): + empty = self.directory / "empty" + empty.mkdir() + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "no manifests found"): + load_targets_matrix.build_matrix(empty) + + def test_rejects_invalid_manifest_filename(self): + (self.directory / "Heimdall_Prod.yml").write_text(HEIMDALL) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "invalid manifest filename"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_manifest_name(self): + (self.directory / "heimdall.yaml").write_text(HEIMDALL) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "duplicate manifest name"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_non_mapping_manifest(self): + (self.directory / "heimdall.yml").write_text("- one\n- two\n") + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must contain a YAML mapping"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_yaml_key(self): + (self.directory / "heimdall.yml").write_text("hosts: [one]\nhosts: [two]\n") + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "duplicate YAML key"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_missing_hosts(self): + (self.directory / "heimdall.yml").write_text(HEIMDALL.replace("hosts: [root@100.75.53.13]\n", "")) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must set hosts"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_missing_app_refs(self): + (self.directory / "heimdall.yml").write_text( + HEIMDALL.replace("app_refs: [rubykatzen/flightdeck@latest]\n", "") + ) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must set app_refs"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_missing_apps(self): + (self.directory / "heimdall.yml").write_text(HEIMDALL.replace("apps:\n traefik: {}\n", "")) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must set apps"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_missing_ssh_private_key_secret(self): + (self.directory / "heimdall.yml").write_text( + HEIMDALL.replace("ssh_private_key_secret: DEPLOY_SSH_PRIVATE_KEY\n", "") + ) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must set ssh_private_key_secret"): + load_targets_matrix.build_matrix(self.directory) + + def test_rejects_missing_sops_age_key_secret(self): + (self.directory / "heimdall.yml").write_text( + HEIMDALL.replace("sops_age_key_secret: HEIMDALL_AGE_PRIVATE_KEY\n", "") + ) + with self.assertRaisesRegex(load_targets_matrix.ManifestError, "must set sops_age_key_secret"): + load_targets_matrix.build_matrix(self.directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/load-vaults-matrix/README.md b/.github/actions/load-vaults-matrix/README.md new file mode 100644 index 0000000..2e3061e --- /dev/null +++ b/.github/actions/load-vaults-matrix/README.md @@ -0,0 +1,18 @@ +# load-vaults-matrix + +Composite GitHub Action that reads every vault manifest in `vaults/` into a GitHub Actions matrix, validating each one's minimal shape. `encrypt-env` still re-parses and validates each manifest itself, in full, when it actually encrypts it - the checks here only catch a structurally broken manifest before it reaches a checkout and dependency install on a different job entirely. Sibling to [`load-targets-matrix`](../load-targets-matrix), same shape, for `targets/*.yml` instead. + +## Usage + +```yaml +- uses: rubykatzen/flightdeck/.github/actions/load-vaults-matrix@main + id: matrix + with: + # directory: vaults # optional, default shown +``` + +The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Each matrix item merges the manifest's own top-level YAML fields with `name` (the file's basename) and `manifest` (its path). + +Every manifest must set `asset` (non-empty string), `keys` (non-empty list), and `env` (non-empty mapping) - just enough to catch a missing or empty section early. `encrypt-env` is what actually validates each field's shape (name patterns, source references, and so on). + +Files may use either the `.yml` or `.yaml` extension. Filenames must match `^[a-z0-9][a-z0-9-]*$` and be unique per directory; duplicate top-level YAML keys within a manifest are rejected. diff --git a/.github/actions/load-yaml-matrix/action.yml b/.github/actions/load-vaults-matrix/action.yml similarity index 61% rename from .github/actions/load-yaml-matrix/action.yml rename to .github/actions/load-vaults-matrix/action.yml index 0b1f6a5..3dcf3e2 100644 --- a/.github/actions/load-yaml-matrix/action.yml +++ b/.github/actions/load-vaults-matrix/action.yml @@ -1,13 +1,10 @@ -name: Load YAML matrix -description: Read every YAML file in a directory into a GitHub Actions matrix. No schema validation. +name: Load vaults matrix +description: Read every vault manifest in a directory into a GitHub Actions matrix. No schema validation - encrypt-env re-parses and validates each manifest itself. inputs: directory: - description: Directory containing YAML files. - required: true - name: - description: Single manifest name to load, or all. + description: Directory containing vault manifests. required: false - default: all + default: vaults outputs: matrix: description: JSON strategy matrix, one item per manifest, merging its parsed YAML fields with name and manifest. @@ -26,6 +23,5 @@ runs: shell: bash env: DIRECTORY: ${{ inputs.directory }} - NAME: ${{ inputs.name }} run: | - python3 "$GITHUB_ACTION_PATH/scripts/load-yaml-matrix.py" --directory "$DIRECTORY" --name "$NAME" + python3 "$GITHUB_ACTION_PATH/scripts/load-vaults-matrix.py" --directory "$DIRECTORY" diff --git a/.github/actions/load-vaults-matrix/requirements.txt b/.github/actions/load-vaults-matrix/requirements.txt new file mode 100644 index 0000000..8392d54 --- /dev/null +++ b/.github/actions/load-vaults-matrix/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.2 diff --git a/.github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py b/.github/actions/load-vaults-matrix/scripts/load-vaults-matrix.py similarity index 79% rename from .github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py rename to .github/actions/load-vaults-matrix/scripts/load-vaults-matrix.py index de13c58..3ceb251 100644 --- a/.github/actions/load-yaml-matrix/scripts/load-yaml-matrix.py +++ b/.github/actions/load-vaults-matrix/scripts/load-vaults-matrix.py @@ -45,13 +45,19 @@ def load_manifest(path): return value -def build_matrix(directory, selected="all"): +def validate_vault(name, manifest): + if not isinstance(manifest.get("asset"), str) or not manifest["asset"]: + raise ManifestError(f"vault {name!r} must set asset to a non-empty string") + if not isinstance(manifest.get("keys"), list) or not manifest["keys"]: + raise ManifestError(f"vault {name!r} must set keys to a non-empty list") + if not isinstance(manifest.get("env"), dict) or not manifest["env"]: + raise ManifestError(f"vault {name!r} must set env to a non-empty mapping") + + +def build_matrix(directory): paths = sorted(directory.glob("*.yml")) + sorted(directory.glob("*.yaml")) if not paths: raise ManifestError(f"no manifests found in {directory}") - names = {path.stem for path in paths} - if selected != "all" and selected not in names: - raise ManifestError(f"unknown name: {selected}") include = [] seen_names = set() for path in paths: @@ -62,8 +68,7 @@ def build_matrix(directory, selected="all"): raise ManifestError(f"duplicate manifest name: {name}") seen_names.add(name) manifest = load_manifest(path) - if selected != "all" and name != selected: - continue + validate_vault(name, manifest) item = {"name": name, "manifest": str(path)} item.update(manifest) include.append(item) @@ -80,10 +85,9 @@ def write_github_output(name, value): def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument("--directory", required=True, type=Path) - parser.add_argument("--name", default="all") args = parser.parse_args(argv) try: - matrix = build_matrix(args.directory, args.name) + matrix = build_matrix(args.directory) encoded = json.dumps(matrix, separators=(",", ":")) write_github_output("matrix", encoded) write_github_output("count", len(matrix["include"])) diff --git a/.github/actions/load-vaults-matrix/tests/test_load_vaults_matrix.py b/.github/actions/load-vaults-matrix/tests/test_load_vaults_matrix.py new file mode 100644 index 0000000..db63574 --- /dev/null +++ b/.github/actions/load-vaults-matrix/tests/test_load_vaults_matrix.py @@ -0,0 +1,96 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-vaults-matrix.py" +SPEC = importlib.util.spec_from_file_location("load_vaults_matrix", MODULE_PATH) +load_vaults_matrix = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(load_vaults_matrix) + + +TRAEFIK = """\ +asset: mainframe-traefik.sops.env +keys: + - mainframe +env: + HTTP_PORT: ${MAINFRAME_TRAEFIK_HTTP_PORT} +""" + +RYBBIT = """\ +asset: mainframe-rybbit.sops.env +keys: + - mainframe +env: + DOMAIN: ${MAINFRAME_DOMAIN} +""" + + +class LoadVaultsMatrixTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.directory = Path(self.temporary_directory.name) + (self.directory / "traefik.yml").write_text(TRAEFIK) + (self.directory / "rybbit.yml").write_text(RYBBIT) + + def test_builds_matrix_from_all_manifests(self): + matrix = load_vaults_matrix.build_matrix(self.directory) + self.assertEqual( + sorted(item["name"] for item in matrix["include"]), + ["rybbit", "traefik"], + ) + + def test_merges_manifest_fields_with_name_and_manifest(self): + matrix = load_vaults_matrix.build_matrix(self.directory) + item = next(item for item in matrix["include"] if item["name"] == "traefik") + self.assertEqual(item["manifest"], str(self.directory / "traefik.yml")) + self.assertEqual(item["asset"], "mainframe-traefik.sops.env") + self.assertEqual(item["keys"], ["mainframe"]) + + def test_rejects_empty_directory(self): + empty = self.directory / "empty" + empty.mkdir() + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "no manifests found"): + load_vaults_matrix.build_matrix(empty) + + def test_rejects_invalid_manifest_filename(self): + (self.directory / "Traefik_Prod.yml").write_text(TRAEFIK) + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "invalid manifest filename"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_manifest_name(self): + (self.directory / "traefik.yaml").write_text(TRAEFIK) + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "duplicate manifest name"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_non_mapping_manifest(self): + (self.directory / "traefik.yml").write_text("- one\n- two\n") + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "must contain a YAML mapping"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_duplicate_yaml_key(self): + (self.directory / "traefik.yml").write_text("keys: [one]\nkeys: [two]\n") + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "duplicate YAML key"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_missing_asset(self): + (self.directory / "traefik.yml").write_text("keys: [mainframe]\nenv: {HTTP_PORT: '80'}\n") + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "must set asset"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_empty_keys(self): + (self.directory / "traefik.yml").write_text( + "asset: mainframe-traefik.sops.env\nkeys: []\nenv: {HTTP_PORT: '80'}\n" + ) + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "must set keys"): + load_vaults_matrix.build_matrix(self.directory) + + def test_rejects_missing_env(self): + (self.directory / "traefik.yml").write_text("asset: mainframe-traefik.sops.env\nkeys: [mainframe]\n") + with self.assertRaisesRegex(load_vaults_matrix.ManifestError, "must set env"): + load_vaults_matrix.build_matrix(self.directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/load-yaml-matrix/README.md b/.github/actions/load-yaml-matrix/README.md deleted file mode 100644 index 7e86bd6..0000000 --- a/.github/actions/load-yaml-matrix/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# load-yaml-matrix - -Composite GitHub Action that reads every YAML file in a directory into a GitHub Actions matrix. It does no schema validation — callers are responsible for the shape of their own manifests. - -## Usage - -```yaml -- uses: rubykatzen/flightdeck/.github/actions/load-yaml-matrix@main - id: matrix - with: - directory: targets # required - # name: all # optional; single manifest name to load, default: all -``` - -The action exposes `matrix`, containing `{ "include": [...] }`, and `count`. Each matrix item merges the manifest's own top-level YAML fields with `name` (the file's basename) and `manifest` (its path). - -Files may use either the `.yml` or `.yaml` extension. Filenames must match `^[a-z0-9][a-z0-9-]*$` and be unique per directory; duplicate top-level YAML keys within a manifest are rejected. Beyond that, the parsed YAML mapping is passed through as-is — validate anything else downstream. diff --git a/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py b/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py deleted file mode 100644 index 8d1cfb2..0000000 --- a/.github/actions/load-yaml-matrix/tests/test_load_yaml_matrix.py +++ /dev/null @@ -1,81 +0,0 @@ -import importlib.util -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "load-yaml-matrix.py" -SPEC = importlib.util.spec_from_file_location("load_yaml_matrix", MODULE_PATH) -load_yaml_matrix = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(load_yaml_matrix) - - -HAWKEYE = """\ -flightdeck_ref: rubykatzen/flightdeck@v1.2.3 -hosts: [rubykatzen-com@100.75.50.2] -""" - -MAINFRAME = """\ -flightdeck_ref: rubykatzen/flightdeck@v1.0.0 -hosts: [deploy@100.64.0.1] -""" - - -class LoadYamlMatrixTest(unittest.TestCase): - def setUp(self): - self.temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary_directory.cleanup) - self.directory = Path(self.temporary_directory.name) - (self.directory / "hawkeye.yml").write_text(HAWKEYE) - (self.directory / "mainframe.yml").write_text(MAINFRAME) - - def test_builds_matrix_from_all_manifests(self): - matrix = load_yaml_matrix.build_matrix(self.directory) - self.assertEqual( - sorted(item["name"] for item in matrix["include"]), - ["hawkeye", "mainframe"], - ) - - def test_merges_manifest_fields_with_name_and_manifest(self): - item = load_yaml_matrix.build_matrix(self.directory, "hawkeye")["include"][0] - self.assertEqual(item["name"], "hawkeye") - self.assertEqual(item["manifest"], str(self.directory / "hawkeye.yml")) - self.assertEqual(item["flightdeck_ref"], "rubykatzen/flightdeck@v1.2.3") - self.assertEqual(item["hosts"], ["rubykatzen-com@100.75.50.2"]) - - def test_filters_selected_manifest(self): - matrix = load_yaml_matrix.build_matrix(self.directory, "hawkeye") - self.assertEqual([item["name"] for item in matrix["include"]], ["hawkeye"]) - - def test_rejects_unknown_name(self): - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "unknown name"): - load_yaml_matrix.build_matrix(self.directory, "missing") - - def test_rejects_empty_directory(self): - empty = self.directory / "empty" - empty.mkdir() - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "no manifests found"): - load_yaml_matrix.build_matrix(empty) - - def test_rejects_invalid_manifest_filename(self): - (self.directory / "Hawkeye_Prod.yml").write_text(HAWKEYE) - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "invalid manifest filename"): - load_yaml_matrix.build_matrix(self.directory) - - def test_rejects_duplicate_manifest_name(self): - (self.directory / "hawkeye.yaml").write_text(HAWKEYE) - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "duplicate manifest name"): - load_yaml_matrix.build_matrix(self.directory) - - def test_rejects_non_mapping_manifest(self): - (self.directory / "hawkeye.yml").write_text("- one\n- two\n") - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "must contain a YAML mapping"): - load_yaml_matrix.build_matrix(self.directory) - - def test_rejects_duplicate_yaml_key(self): - (self.directory / "hawkeye.yml").write_text("hosts: [one]\nhosts: [two]\n") - with self.assertRaisesRegex(load_yaml_matrix.ManifestError, "duplicate YAML key"): - load_yaml_matrix.build_matrix(self.directory) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/actions/renovate/README.md b/.github/actions/renovate/README.md new file mode 100644 index 0000000..48d2bd5 --- /dev/null +++ b/.github/actions/renovate/README.md @@ -0,0 +1,36 @@ +# renovate + +Composite GitHub Action that re-pulls and recreates one-or-more apps' containers on a target already present in the caller's own checkout, without touching versions - no new app bundle, no new vault-sourced env, no rebuilt release tree. Just `docker compose pull && docker compose up -d` per requested app, against its already-current release, on each of the target's hosts. Sibling to [`deploy`](../deploy), same shape, deliberately narrower job. + +## Usage + +```yaml +jobs: + renovate: + strategy: + matrix: ${{ fromJson(needs.load-targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: rubykatzen/flightdeck/.github/actions/renovate@main + id: renovate + with: + apps: ${{ inputs.apps }} # required, JSON array e.g. '["traefik","rybbit"]' + target-manifest: ${{ matrix.manifest }} # required, path in this repository + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + # tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) + # tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # required only if tailscale-oauth-client-id is set + - name: Notify Telegram + if: steps.renovate.outputs.updated == 'true' + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Renovate: updated on ${{ steps.renovate.outputs.target-name }} (${{ steps.renovate.outputs.updated-hosts }})" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} +``` + +Not every target runs every requested app - `renovate.py` decides that itself, from the target manifest's own `apps` mapping, and simply does nothing (never opening an SSH connection) if none of the requested apps are present. To tell whether a host's image actually changed (rather than the pull being a no-op), it compares `docker compose images -q` output before and after the pull, and exposes that as this action's own `updated`/`updated-hosts`/`target-name` outputs. + +Unlike [`deploy`](../deploy) - which has no notification logic at all - this action still has none of its own either, on purpose: it only ever reports whether something changed. Sending a Telegram message (or anything else) is the *caller's* job, as a separate step reading these outputs, same as shown above - a different caller is free to wire up a different channel, or none, without forking this action. + +Same checkout model as [`deploy`](../deploy): this action never checks out anything itself, it just reads `target-manifest` from whatever the caller's own preceding `actions/checkout` step already put on disk. diff --git a/.github/actions/renovate/action.yml b/.github/actions/renovate/action.yml new file mode 100644 index 0000000..7d14b3e --- /dev/null +++ b/.github/actions/renovate/action.yml @@ -0,0 +1,64 @@ +name: Renovate +description: Re-pull and recreate one-or-more apps' containers on a target already present in the caller's own checkout, without touching versions. +inputs: + apps: + description: JSON array of app name(s) to renovate (re-pull their image and recreate their containers). Empty array means every app this target runs. + required: false + default: "[]" + target-manifest: + description: Path to the target manifest file, e.g. targets/heimdall.yml - already checked out by the caller. + required: true + ssh-private-key: + description: SSH private key used to connect to the hosts. + required: true + tailscale-oauth-client-id: + description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). + required: false + default: "" + tailscale-oauth-secret: + description: Tailscale OAuth client secret used to join the tailnet. Required only when tailscale-oauth-client-id is set. + required: false + default: "" + tailscale-tags: + description: Comma-separated ACL tags for the ephemeral tailnet node. + required: false + default: tag:ci +outputs: + updated: + description: "'true' if at least one requested app's image actually changed on this target." + value: ${{ steps.run.outputs.updated }} + updated-hosts: + description: Comma-separated app@host pairs that were actually updated. + value: ${{ steps.run.outputs.updated_hosts }} + target-name: + description: This target's name, derived from the manifest's own filename. + value: ${{ steps.run.outputs.target_name }} +runs: + using: composite + steps: + - name: Install deploy dependencies + shell: bash + run: python3 -m pip install --disable-pip-version-check --requirement "$GITHUB_ACTION_PATH/requirements.txt" + - uses: tailscale/github-action@v4 + if: inputs.tailscale-oauth-client-id != '' + with: + oauth-client-id: ${{ inputs.tailscale-oauth-client-id }} + oauth-secret: ${{ inputs.tailscale-oauth-secret }} + tags: ${{ inputs.tailscale-tags }} + - name: Load SSH key + shell: bash + run: | + eval "$(ssh-agent -s)" + echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" + echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" + ssh-add - <<< "${{ inputs.ssh-private-key }}" + - name: Run renovate + id: run + shell: bash + env: + APPS: ${{ inputs.apps }} + TARGET_MANIFEST: ${{ inputs.target-manifest }} + run: | + jq -n --argjson apps "$APPS" --arg target_manifest "$TARGET_MANIFEST" \ + '{apps: $apps, target_manifest: $target_manifest}' \ + | python3 "$GITHUB_ACTION_PATH/renovate.py" diff --git a/.github/actions/renovate/renovate.py b/.github/actions/renovate/renovate.py new file mode 100644 index 0000000..3702271 --- /dev/null +++ b/.github/actions/renovate/renovate.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Renovate: re-pull and recreate one-or-more apps' containers on one +target, read directly from that target's own targets/*.yml manifest, +without touching versions - no new app bundle, no new vault-sourced env, +no rebuilt release tree. Just `docker compose pull && docker compose +up -d` per requested app, against its already-current release, on each +of the target's hosts. + +Like deploy.py, this takes a manifest *path* and parses it itself - the +same shape encrypt-env's render-env.py already uses for vault manifests, +rather than flattening a target's hosts/apps/path into separate inputs +the caller has to build. Finding every target and resolving its secrets +happens in the calling workflow's own matrix (see README's "renovate" +section), not here - this script never reads a directory or touches +GitHub Secrets itself. + +Reads a JSON config from stdin: {"apps": ["traefik", "rybbit"], +"target_manifest": "targets/heimdall.yml"}. Any requested app that isn't +a key in that manifest's own `apps` mapping is skipped, not an error - a +target-matrix fan-out dispatches to every target regardless of which of +the requested apps it actually runs; skipping all of them there is a +clean no-op. An empty `apps` list means "every app this target runs" - +the scheduled nightly run has no specific apps to name, so it renovates +everything. + +Writes `updated`/`updated_hosts`/`target_name` to $GITHUB_OUTPUT so the +calling workflow can notify only when a host's image actually changed, +rather than on every run. `updated_hosts` lists `app@host` pairs, since +more than one app may have been renovated in the same run. +""" +import json +import os +import shlex +import sys +from pathlib import Path + +import paramiko +import yaml +from fabric import Connection + + +def expand_home(path, home): + return home + path[1:] if path.startswith("~") else path + + +def renovate_host(host, base_path, app): + connection = Connection(host) + connection.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + home = connection.run("echo $HOME", hide=True).stdout.strip() + compose_dir = shlex.quote(f"{expand_home(base_path, home)}/current/apps/{app}") + command = ( + f"cd {compose_dir} && before=$(docker compose images -q) && docker compose pull " + f'&& after=$(docker compose images -q) && docker compose up -d --remove-orphans ' + f'&& if [ "$before" != "$after" ]; then echo RENOVATE_UPDATED; fi' + ) + result = connection.run(command) + return "RENOVATE_UPDATED" in result.stdout + + +def write_github_output(name, value): + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + output.write(f"{name}={value}\n") + + +def main(): + config = json.load(sys.stdin) + requested_apps = config["apps"] + manifest_path = Path(config["target_manifest"]) + target_name = manifest_path.stem + write_github_output("target_name", target_name) + + target = yaml.safe_load(manifest_path.read_text()) + target_apps = target.get("apps") or {} + matching_apps = [app for app in requested_apps if app in target_apps] if requested_apps else list(target_apps) + if not matching_apps: + if requested_apps: + print(f"none of {requested_apps} are deployed on target {target_name!r}, skipping") + else: + print(f"target {target_name!r} has no apps deployed, skipping") + write_github_output("updated", "false") + return + + base_path = target.get("path", "~/flightdeck") + updated = [] + for app in matching_apps: + for host in target["hosts"]: + print(f"Renovating {app} on {host}") + if renovate_host(host, base_path, app): + updated.append(f"{app}@{host}") + + write_github_output("updated", "true" if updated else "false") + write_github_output("updated_hosts", ",".join(updated)) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/renovate/requirements.txt b/.github/actions/renovate/requirements.txt new file mode 100644 index 0000000..f38c8dc --- /dev/null +++ b/.github/actions/renovate/requirements.txt @@ -0,0 +1,2 @@ +fabric +PyYAML==6.0.3 diff --git a/.github/actions/renovate/tests/test_renovate.py b/.github/actions/renovate/tests/test_renovate.py new file mode 100644 index 0000000..af066d5 --- /dev/null +++ b/.github/actions/renovate/tests/test_renovate.py @@ -0,0 +1,178 @@ +import importlib.util +import io +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +DEPLOY_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(DEPLOY_DIR)) + +MODULE_PATH = DEPLOY_DIR / "renovate.py" +SPEC = importlib.util.spec_from_file_location("renovate_entrypoint", MODULE_PATH) +renovate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(renovate) + + +class FakeConnection: + """Stand-in for fabric.Connection - records commands instead of opening + a real SSH session, mirroring test_deploy.py's FakeConnection.""" + + def __init__(self, host, pull_stdout=""): + self.host = host + self.client = SimpleNamespace(set_missing_host_key_policy=lambda policy: None) + self.commands = [] + self.pull_stdout = pull_stdout + + def run(self, command, hide=False): + self.commands.append(command) + if command == "echo $HOME": + return SimpleNamespace(stdout="/home/deploy\n") + return SimpleNamespace(stdout=self.pull_stdout) + + +class ExpandHomeTest(unittest.TestCase): + def test_expands_tilde_prefix(self): + self.assertEqual(renovate.expand_home("~/flightdeck", "/home/deploy"), "/home/deploy/flightdeck") + + def test_leaves_absolute_path_untouched(self): + self.assertEqual(renovate.expand_home("/opt/flightdeck", "/home/deploy"), "/opt/flightdeck") + + +class RenovateHostTest(unittest.TestCase): + def test_pulls_and_recreates_without_touching_the_release(self): + fake = FakeConnection("deploy@host") + with patch.object(renovate, "Connection", return_value=fake): + renovate.renovate_host("deploy@host", "~/flightdeck", "beszel") + + joined = "\n".join(fake.commands) + self.assertIn("cd /home/deploy/flightdeck/current/apps/beszel", joined) + self.assertIn("before=$(docker compose images -q)", joined) + self.assertIn("docker compose pull", joined) + self.assertIn("after=$(docker compose images -q)", joined) + self.assertIn("docker compose up -d --remove-orphans", joined) + + def test_returns_true_when_the_image_changed(self): + fake = FakeConnection("deploy@host", pull_stdout="RENOVATE_UPDATED\n") + with patch.object(renovate, "Connection", return_value=fake): + self.assertTrue(renovate.renovate_host("deploy@host", "~/flightdeck", "beszel")) + + def test_returns_false_when_the_image_was_already_current(self): + fake = FakeConnection("deploy@host", pull_stdout="") + with patch.object(renovate, "Connection", return_value=fake): + self.assertFalse(renovate.renovate_host("deploy@host", "~/flightdeck", "beszel")) + + +class MainTest(unittest.TestCase): + def _run_main(self, directory, apps, manifest_text, manifest_name="heimdall"): + manifest_path = Path(directory) / f"{manifest_name}.yml" + manifest_path.write_text(manifest_text) + output_path = Path(directory) / "outputs" + + config = {"apps": apps, "target_manifest": str(manifest_path)} + with ( + patch.object(sys, "stdin", io.StringIO(json.dumps(config))), + patch.dict(os.environ, {"GITHUB_OUTPUT": str(output_path)}), + ): + renovate.main() + return output_path.read_text() if output_path.exists() else "" + + def test_skips_a_target_that_runs_none_of_the_requested_apps(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host") as fake_renovate_host, + ): + outputs = self._run_main(directory, ["beszel"], "apps:\n traefik: {}\nhosts: [deploy@host]\n") + + fake_renovate_host.assert_not_called() + self.assertIn("updated=false\n", outputs) + self.assertIn("target_name=heimdall\n", outputs) + + def test_renovates_every_app_when_apps_is_empty(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=False) as fake_renovate_host, + ): + self._run_main(directory, [], "apps:\n traefik: {}\n gatus: {}\nhosts: [deploy@host]\n") + + fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "traefik") + fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "gatus") + self.assertEqual(fake_renovate_host.call_count, 2) + + def test_skips_a_target_with_no_apps_when_apps_is_empty(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host") as fake_renovate_host, + ): + outputs = self._run_main(directory, [], "apps: {}\nhosts: [deploy@host]\n") + + fake_renovate_host.assert_not_called() + self.assertIn("updated=false\n", outputs) + + def test_renovates_only_the_requested_apps_present_on_the_target(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=False) as fake_renovate_host, + ): + self._run_main( + directory, + ["traefik", "rybbit", "beszel"], + "apps:\n traefik: {}\n gatus: {}\nhosts: [deploy@host]\n", + ) + + fake_renovate_host.assert_called_once_with("deploy@host", "~/flightdeck", "traefik") + + def test_reports_updated_app_host_pairs_when_the_image_changed(self): + def fake_renovate_host(host, base_path, app): + return app == "beszel" and host == "deploy@app1.example.com" + + manifest = ( + "apps:\n beszel: {}\n traefik: {}\n" + "hosts: [deploy@app1.example.com, deploy@app2.example.com]\npath: ~/flightdeck\n" + ) + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", side_effect=fake_renovate_host), + ): + outputs = self._run_main(directory, ["beszel", "traefik"], manifest) + + self.assertIn("updated=true\n", outputs) + self.assertIn("updated_hosts=beszel@deploy@app1.example.com\n", outputs) + + def test_reports_not_updated_when_every_host_was_already_current(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=False), + ): + outputs = self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n") + + self.assertIn("updated=false\n", outputs) + self.assertIn("updated_hosts=\n", outputs) + + def test_defaults_path_when_omitted(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=False) as fake_renovate_host, + ): + self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n") + + fake_renovate_host.assert_called_once_with("deploy@host", "~/flightdeck", "beszel") + + def test_derives_target_name_from_manifest_filename(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=False), + ): + outputs = self._run_main( + directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n", manifest_name="mainframe" + ) + + self.assertIn("target_name=mainframe\n", outputs) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/deploy-shared.yml b/.github/workflows/deploy-shared.yml deleted file mode 100644 index c2e84a9..0000000 --- a/.github/workflows/deploy-shared.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Deploy (shared) -on: - workflow_call: - inputs: - hosts: - description: JSON array of user@host SSH destinations to deploy to. - type: string - required: true - app-refs: - description: JSON array of release refs for app bundles to merge into the release, in owner/repo@tag[:asset] format. - type: string - required: true - apps: - description: JSON object mapping each app name to run on this target to its own env_refs list, e.g. {"traefik":{"env_refs":["owner/repo@latest:traefik.sops.env"]}}. - type: string - required: true - path: - description: Base path on the target host for releases, shared files, and the current symlink. - type: string - default: "~/flightdeck" - keep-releases: - description: Number of past releases to keep on the target host. - type: number - default: 5 - tailscale-oauth-client-id: - description: Tailscale OAuth client ID used to join the tailnet. Leave unset to skip joining a tailnet (e.g. when the runner already has network access to the hosts). - type: string - default: "" - tailscale-tags: - description: Comma-separated ACL tags for the ephemeral tailnet node. - type: string - default: tag:ci - secrets: - ssh-private-key: - description: SSH private key used to connect to the hosts. - required: true - sops-age-key: - description: Private SOPS age key used to decrypt this target's vault-sourced env, on the runner. - required: true - tailscale-oauth-secret: - description: Tailscale OAuth client secret used to join the tailnet. Required only when tailscale-oauth-client-id is set. - required: false -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - repository: ${{ job.workflow_repository }} - ref: ${{ job.workflow_sha }} - - name: Install deploy dependencies - shell: bash - run: pip install --user --break-system-packages -r deploy/requirements.txt - - name: Install sops - shell: bash - env: - SOPS_VERSION: "3.13.1" - run: | - curl --fail --location --silent --show-error \ - "https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64" \ - --output /usr/local/bin/sops - chmod +x /usr/local/bin/sops - - uses: tailscale/github-action@v4 - if: inputs.tailscale-oauth-client-id != '' - with: - oauth-client-id: ${{ inputs.tailscale-oauth-client-id }} - oauth-secret: ${{ secrets.tailscale-oauth-secret }} - tags: ${{ inputs.tailscale-tags }} - - name: Load SSH key - shell: bash - run: | - eval "$(ssh-agent -s)" - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" - echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - ssh-add - <<< "${{ secrets.ssh-private-key }}" - - name: Run deploy - shell: bash - env: - HOSTS: ${{ inputs.hosts }} - APP_REFS: ${{ inputs.app-refs }} - APPS: ${{ inputs.apps }} - DEPLOY_PATH: ${{ inputs.path }} - KEEP_RELEASES: ${{ inputs.keep-releases }} - SOPS_AGE_KEY: ${{ secrets.sops-age-key }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - jq -n \ - --argjson hosts "$HOSTS" \ - --argjson app_refs "$APP_REFS" \ - --argjson apps "$APPS" \ - --arg path "$DEPLOY_PATH" \ - --argjson keep_releases "$KEEP_RELEASES" \ - --arg sops_age_key "$SOPS_AGE_KEY" \ - '{hosts: $hosts, app_refs: $app_refs, apps: $apps, path: $path, keep_releases: $keep_releases, sops_age_key: $sops_age_key}' \ - | python3 deploy/deploy.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a605a96..dc25444 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,38 +1,36 @@ name: Deploy on: - workflow_dispatch: - inputs: - target: - description: Deploy target to redeploy. Use "all" to redeploy every target. - type: string - default: all + workflow_dispatch: {} + workflow_call: {} jobs: - deploy-targets: + load-targets: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - count: ${{ steps.matrix.outputs.count }} + matrix: ${{ steps.load-targets.outputs.matrix }} + count: ${{ steps.load-targets.outputs.count }} steps: - uses: actions/checkout@v7 - - uses: $/.github/actions/load-yaml-matrix - id: matrix - with: - directory: targets - name: ${{ inputs.target || 'all' }} + - uses: $/.github/actions/load-targets-matrix + id: load-targets deploy: - needs: deploy-targets - if: needs.deploy-targets.outputs.count != '0' + needs: load-targets + if: needs.load-targets.outputs.count != '0' strategy: - matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} - uses: $/.github/workflows/deploy-shared.yml - with: - hosts: ${{ toJson(matrix.hosts) }} - app-refs: ${{ toJson(matrix.app_refs) }} - apps: ${{ toJson(matrix.apps) }} - path: ${{ matrix.path || '~/flightdeck' }} - keep-releases: ${{ matrix.keep_releases || 5 }} - tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} - secrets: - ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} - tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} - sops-age-key: ${{ secrets[matrix.credentials.secrets.sops_age_key] }} + matrix: ${{ fromJson(needs.load-targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: $/.github/actions/deploy + with: + target-manifest: ${{ matrix.manifest }} + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + sops-age-key: ${{ secrets[matrix.sops_age_key_secret] }} + github-token: ${{ secrets.GITHUB_TOKEN }} + tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} + tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} + - name: Notify Telegram + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Deploy: ${{ matrix.name }} updated" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bf2817..14c5fd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,26 +29,24 @@ jobs: with: release-tag: ${{ needs.release.outputs.tag_name }} token: ${{ secrets.GITHUB_TOKEN }} - encrypt-vaults: + load-vaults: needs: release if: needs.release.outputs.release_created == 'true' runs-on: ubuntu-latest outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - count: ${{ steps.matrix.outputs.count }} + matrix: ${{ steps.load-vaults.outputs.matrix }} + count: ${{ steps.load-vaults.outputs.count }} steps: - uses: actions/checkout@v7 with: ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-yaml-matrix - id: matrix - with: - directory: vaults + - uses: $/.github/actions/load-vaults-matrix + id: load-vaults encrypt: - needs: [release, encrypt-vaults] - if: needs.encrypt-vaults.outputs.count != '0' + needs: [release, load-vaults] + if: needs.load-vaults.outputs.count != '0' strategy: - matrix: ${{ fromJson(needs.encrypt-vaults.outputs.matrix) }} + matrix: ${{ fromJson(needs.load-vaults.outputs.matrix) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -62,35 +60,8 @@ jobs: env: GITHUB_SECRETS_JSON: ${{ toJson(secrets) }} GITHUB_VARS_JSON: ${{ toJson(vars) }} - deploy-targets: - needs: release - if: needs.release.outputs.release_created == 'true' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - count: ${{ steps.matrix.outputs.count }} - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.release.outputs.tag_name }} - - uses: $/.github/actions/load-yaml-matrix - id: matrix - with: - directory: targets deploy: - needs: [upload-apps, encrypt, deploy-targets] - if: needs.deploy-targets.outputs.count != '0' - strategy: - matrix: ${{ fromJson(needs.deploy-targets.outputs.matrix) }} - uses: $/.github/workflows/deploy-shared.yml - with: - hosts: ${{ toJson(matrix.hosts) }} - app-refs: ${{ toJson(matrix.app_refs) }} - apps: ${{ toJson(matrix.apps) }} - path: ${{ matrix.path || '~/flightdeck' }} - keep-releases: ${{ matrix.keep_releases || 5 }} - tailscale-oauth-client-id: ${{ vars[matrix.credentials.variables.tailscale_oauth_client_id] }} - secrets: - ssh-private-key: ${{ secrets[matrix.credentials.secrets.ssh_private_key] }} - tailscale-oauth-secret: ${{ secrets[matrix.credentials.secrets.tailscale_oauth_secret] }} - sops-age-key: ${{ secrets[matrix.credentials.secrets.sops_age_key] }} + needs: [release, upload-apps, encrypt] + if: needs.release.outputs.release_created == 'true' + uses: $/.github/workflows/deploy.yml + secrets: inherit diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml new file mode 100644 index 0000000..b5056c8 --- /dev/null +++ b/.github/workflows/renovate.yml @@ -0,0 +1,44 @@ +name: Renovate +on: + workflow_dispatch: + inputs: + apps: + description: JSON array of app name(s) to renovate (re-pull their image and recreate their containers) wherever each is currently deployed, e.g. ["traefik","rybbit"]. Leave empty to renovate every app on every target. + type: string + required: false + default: "[]" + schedule: + - cron: "0 3 * * *" +jobs: + load-targets: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load-targets.outputs.matrix }} + count: ${{ steps.load-targets.outputs.count }} + steps: + - uses: actions/checkout@v7 + - uses: $/.github/actions/load-targets-matrix + id: load-targets + renovate: + needs: load-targets + if: needs.load-targets.outputs.count != '0' + strategy: + matrix: ${{ fromJson(needs.load-targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: $/.github/actions/renovate + id: renovate + with: + apps: ${{ inputs.apps || '[]' }} + target-manifest: ${{ matrix.manifest }} + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} + tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} + - name: Notify Telegram + if: steps.renovate.outputs.updated == 'true' + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Renovate: updated on ${{ steps.renovate.outputs.target-name }} (${{ steps.renovate.outputs.updated-hosts }})" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} diff --git a/AGENTS.md b/AGENTS.md index 193c314..860ea2e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,7 @@ input. ## Repository Overview -This is a Docker-based deployment system (flightdeck) that manages core services via docker-compose. The architecture uses Traefik as a reverse proxy with automatic SSL certificate management, and can merge optional extra application catalogs during deploy. There is no manual administration flow at all - no server-side console access, no local quick-start. Every deploy goes through `targets/`/`vaults/` manifests and GitHub Actions (`deploy/deploy.py`); see the "Environment Variable System" and "CI/CD" sections below. +This is a Docker-based deployment system (flightdeck) that manages core services via docker-compose. The architecture uses Traefik as a reverse proxy with automatic SSL certificate management, and can merge optional extra application catalogs during deploy. There is no manual administration flow at all - no server-side console access, no local quick-start. Every deploy goes through `targets/`/`vaults/` manifests and GitHub Actions (`.github/actions/deploy/deploy.py`); see the "Environment Variable System" and "CI/CD" sections below. ## Core Architecture @@ -60,7 +60,7 @@ This is a Docker-based deployment system (flightdeck) that manages core services - `apps/` - Contains core docker-compose configurations and shared compose templates - `apps-data/` - Persistent data storage on the target host (not in this repo): only state that must survive across releases and isn't regenerated by a deploy (database volumes, `traefik/acme.json`) -- `deploy/` - The push-based deploy entrypoint and its supporting modules (ref resolution, collision detection, decryption, config rendering), run on the GitHub Actions runner +- `.github/actions/deploy/` - The push-based deploy entrypoint and its supporting modules (ref resolution, collision detection, decryption, config rendering), run on the GitHub Actions runner; `.github/actions/renovate/` is its narrower sibling (re-pull/recreate one-or-more apps' containers without touching versions) ### Docker Compose Architecture @@ -90,7 +90,7 @@ The repository uses a modular docker-compose structure with reusable components: Each app in `apps/` has: - `docker-compose.yml` extending common services - Optional `*.tpl` config files sitting directly next to `docker-compose.yml` (see "Config Templates" below) - - A `.env` on the target host only, decrypted and placed there by `deploy/deploy.py` (never checked into this repo, never present until a real deploy runs) + - A `.env` on the target host only, decrypted and placed there by `.github/actions/deploy/deploy.py` (never checked into this repo, never present until a real deploy runs) ### Healthchecks @@ -105,11 +105,11 @@ Standard shape used throughout the catalog: `start_period: 30s`, `interval: 30s` ### Environment Variable System -There is no root `.env` anywhere - not on a target host, not locally. Each app's env comes entirely from that app's own vault(s), declared in `targets/{target}.yml`'s `apps..env_refs` (see README's "Vaults And Targets"). `env_refs` is optional - an app with no vault-sourced values at all (e.g. `apps/beszel/docker-compose.yml`, which reads nothing but `APP_NAME`/`DATA_DIR`) just omits it, rather than pointing at a vault manifest with an empty `env:`. `deploy/deploy.py` runs on the GitHub Actions runner: it downloads each app's still-encrypted vault assets, checks their key names for collisions from the ciphertext directly (no decryption needed for that check), decrypts them with the target's private SOPS age key, concatenates the plaintext, and writes it straight into that app's `.env` in the release tree before pushing. `deploy/render.py`'s `render_template` then does the same substitution `envsubst` would, also on the runner, for that app's `*.tpl` files, using the just-decrypted values (see "Config Templates" below). +There is no root `.env` anywhere - not on a target host, not locally. Each app's env comes entirely from that app's own vault(s), declared in `targets/{target}.yml`'s `apps..env_refs` (see README's "Vaults And Targets"). `env_refs` is optional - an app with no vault-sourced values at all (e.g. `apps/beszel/docker-compose.yml`, which reads nothing but `APP_NAME`/`DATA_DIR`) just omits it, rather than pointing at a vault manifest with an empty `env:`. `.github/actions/deploy/deploy.py` runs on the GitHub Actions runner: it downloads each app's still-encrypted vault assets, checks their key names for collisions from the ciphertext directly (no decryption needed for that check), decrypts them with the target's private SOPS age key, concatenates the plaintext, and writes it straight into that app's `.env` in the release tree before pushing. `.github/actions/deploy/render.py`'s `render_template` then does the same substitution `envsubst` would, also on the runner, for that app's `*.tpl` files, using the just-decrypted values (see "Config Templates" below). A vault declares the exact final variable name an app receives directly (e.g. `HTTP_PORT`, not `TRAEFIK_HTTP_PORT`) - there is no automatic prefix-stripping or filtering step anywhere. Variables for one app are never visible to another app, since each app's `.env` is built from that app's own vault(s) only. This allows running docker compose directly from the app folder without any `--env-file` flags while keeping app secrets scoped. -Two variables in every app's `.env` are never vault-sourced - `deploy/deploy.py` computes and writes them itself: +Two variables in every app's `.env` are never vault-sourced - `.github/actions/deploy/deploy.py` computes and writes them itself: - `APP_NAME` - just the app's own directory name (`traefik`, `rybbit`, ...). Written by `resolve_app_envs` on the runner, identical for every host a target deploys to, so it lives in the release tree's static `.env` like anything else. - `DATA_DIR` - the absolute, persistent data path for that app on the *target host* (`{base_path}/apps-data/{app}`). This can't be computed on the runner: `base_path` comes from a target's `path` (default `~/flightdeck`), and `~` only resolves once connected to a specific host (`deploy_to_host` reads `$HOME` over SSH) - and a target's `hosts:` can list more than one, potentially with different home directories. So `DATA_DIR` is appended to each app's already-pushed `.env` on the host itself, inside `deploy_to_host`'s per-app loop, after the release is extracted but before `docker compose` ever runs - never baked into the release tree the way `APP_NAME` is. @@ -122,7 +122,7 @@ Compose files reference `${DATA_DIR}` directly (`${DATA_DIR}/postgres:/var/lib/p The mechanism is pure naming convention, no manifest or registration needed - the same `.tpl` marker Terraform's `templatefile()` uses, as a terminal suffix (`codecov.yml.tpl`, same placement as Terraform's `user_data.tpl`). Note this means editors and GitHub's diff view won't apply YAML syntax highlighting to the template out of the box (they pick a language by the last extension, and `.tpl` isn't a registered one anywhere by default) - the rendered output (`codecov.yml`) isn't affected, only the template source. Configure a file association per editor if that matters to you (e.g. Zed's `file_types` setting). -Any file directly inside `apps/{app}/` (next to `docker-compose.yml`, no special subdirectory) matching `*.tpl` is a template. `deploy/deploy.py`'s `render_app_configs` finds them with a plain glob, substitutes with `deploy/render.py` (an `envsubst`-equivalent - `$VAR`/`${VAR}` only, no bash `${VAR:-default}` fallback syntax, missing variable becomes an empty string), and writes the result as a sibling file in the same directory with `.tpl` stripped (`codecov.yml.tpl` → `codecov.yml`), `chmod 600` since rendered output can carry secrets. This happens on the runner, before the release is archived, so the rendered file rides inside the release tar next to `.env` and is versioned with that release like everything else - never written directly onto the target host outside the atomic release/symlink-switch step. +Any file directly inside `apps/{app}/` (next to `docker-compose.yml`, no special subdirectory) matching `*.tpl` is a template. `.github/actions/deploy/deploy.py`'s `render_app_configs` finds them with a plain glob, substitutes with `.github/actions/deploy/render.py` (an `envsubst`-equivalent - `$VAR`/`${VAR}` only, no bash `${VAR:-default}` fallback syntax, missing variable becomes an empty string), and writes the result as a sibling file in the same directory with `.tpl` stripped (`codecov.yml.tpl` → `codecov.yml`), `chmod 600` since rendered output can carry secrets. This happens on the runner, before the release is archived, so the rendered file rides inside the release tar next to `.env` and is versioned with that release like everything else - never written directly onto the target host outside the atomic release/symlink-switch step. Compose files mount the rendered file by its plain relative path (`./codecov.yml:/config/codecov.yml`), one line per file - not a whole-directory mount - so it's obvious from the compose file alone which container path each config file lands at. `apps-data/{app}/` stays reserved for the opposite case: state a deploy must never regenerate (`acme.json`, database data directories) - never templated output. @@ -197,7 +197,7 @@ HTTP-01 (`httpChallenge`) is always configured and needs nothing from the vault ## Operations -There are no wrapper scripts and nothing runs them - starting, stopping, and restarting apps all happen by deploying (`deploy/deploy.py`, see "CI/CD" below). The external Docker networks `traefik` and `databases` are created idempotently by `deploy/deploy.py` on every deploy (derived from `apps/networks.yml`'s `external: true` entries), not by a separate first-run step. `apps-data/traefik/acme.json` isn't created by `deploy/deploy.py` at all - traefik's `docker-compose.yml` mounts `apps-data/{app}/` as a directory (not the file directly, which would make Docker create a directory in its place if the file doesn't exist yet), and Traefik creates `acme.json` inside it itself on first start, with the permissions it requires. +There are no wrapper scripts and nothing runs them - starting, stopping, and restarting apps all happen by deploying (`.github/actions/deploy/deploy.py`, see "CI/CD" below). The external Docker networks `traefik` and `databases` are created idempotently by `.github/actions/deploy/deploy.py` on every deploy (derived from `apps/networks.yml`'s `external: true` entries), not by a separate first-run step. `apps-data/traefik/acme.json` isn't created by `.github/actions/deploy/deploy.py` at all - traefik's `docker-compose.yml` mounts `apps-data/{app}/` as a directory (not the file directly, which would make Docker create a directory in its place if the file doesn't exist yet), and Traefik creates `acme.json` inside it itself on first start, with the permissions it requires. Debugging an already-deployed app means SSHing into the target host directly and using Docker Compose itself - no wrapper needed, since each app's directory is already a complete, ready-to-run Compose project (real `.env` sitting next to the compose file): @@ -389,14 +389,14 @@ GitHub Actions workflow (`.github/workflows/release.yml`) manages releases via [ 1. On every push to `main`, Release Please opens/updates a `chore: release X.Y.Z` PR with the computed version and generated `CHANGELOG.md` entry 2. Merging that PR tags the release and publishes a GitHub Release -3. A job then builds and uploads `flightdeck-apps.zip` from the `apps/` catalog. Deploy refs may use `@latest` as an alias resolved through GitHub's latest release API (`deploy/resolve.py`); no mutable `latest` release/tag is created. +3. A job then builds and uploads `flightdeck-apps.zip` from the `apps/` catalog. Deploy refs may use `@latest` as an alias resolved through GitHub's latest release API (`.github/actions/deploy/resolve.py`); no mutable `latest` release/tag is created. -Deployment helpers live in this repository, entirely under `deploy/`, run only on the GitHub Actions runner - the target host never runs any of this: +Deployment helpers live in this repository, entirely under `.github/actions/deploy/` (and `.github/actions/renovate/` for its narrower sibling), run only on the GitHub Actions runner - the target host never runs any of this: -- `deploy/deploy.py` is the deploy entrypoint. It resolves and downloads every ref in `app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit) and merges them into a release tree locally; for each app in the target's `apps` mapping, downloads its `env_refs` (still encrypted), decrypts them with the target's private SOPS age key, writes the plaintext into that app's `.env` in the release tree, and renders that app's `*.tpl` files in place with the decrypted values (see "Config Templates" above). It writes a `manifest.json` into the release tree (`schema_version`, `release` timestamp, resolved `app_refs`/`env_refs` - the actual tag `resolve.py` pulled, never `@latest` - and the desired `apps` list; no secrets, no target identifier, since whoever's reading it is already on that specific host). It then opens an SSH connection per host, pushes the finished release as one tarball (real `.env`, already-rendered config, the manifest, all versioned together), bootstraps networks/directories idempotently, reads the *previous* release's `manifest.json` off `current` and `docker compose down`s any app present there but no longer in the desired set (using `current` before it moves, so that app's last-known compose file/`.env` are still intact - safe even though every release tree already contains every app's compose file regardless of whether the target wants it, since `build_release` copies the whole catalog every time), switches `current` to the new timestamped release, and runs `docker compose pull && docker compose up -d` per desired app directly (no wrapper script on the host at all). -- `deploy/resolve.py`, `deploy/collisions.py`, `deploy/vault.py`, and `deploy/render.py` hold, respectively, the ref-resolution, ciphertext collision-detection, decryption, and template-rendering logic - each with real `unittest` coverage in `deploy/tests/`. +- `.github/actions/deploy/deploy.py` is the deploy entrypoint. It resolves and downloads every ref in `app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit) and merges them into a release tree locally; for each app in the target's `apps` mapping, downloads its `env_refs` (still encrypted), decrypts them with the target's private SOPS age key, writes the plaintext into that app's `.env` in the release tree, and renders that app's `*.tpl` files in place with the decrypted values (see "Config Templates" above). It writes a `manifest.json` into the release tree (`schema_version`, `release` timestamp, resolved `app_refs`/`env_refs` - the actual tag `resolve.py` pulled, never `@latest` - and the desired `apps` list; no secrets, no target identifier, since whoever's reading it is already on that specific host). It then opens an SSH connection per host, pushes the finished release as one tarball (real `.env`, already-rendered config, the manifest, all versioned together), bootstraps networks/directories idempotently, reads the *previous* release's `manifest.json` off `current` and `docker compose down`s any app present there but no longer in the desired set (using `current` before it moves, so that app's last-known compose file/`.env` are still intact - safe even though every release tree already contains every app's compose file regardless of whether the target wants it, since `build_release` copies the whole catalog every time), switches `current` to the new timestamped release, and runs `docker compose pull && docker compose up -d` per desired app directly (no wrapper script on the host at all). +- `.github/actions/deploy/resolve.py`, `.github/actions/deploy/collisions.py`, `.github/actions/deploy/vault.py`, and `.github/actions/deploy/render.py` hold, respectively, the ref-resolution, ciphertext collision-detection, decryption, and template-rendering logic - each with real `unittest` coverage in `.github/actions/deploy/tests/`. - `.github/actions/encrypt-env/` is a local composite action for rendering `vaults/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset — vault manifests hold only env/secrets, not app selection -- `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `deploy/deploy.py` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository +- `.github/actions/deploy/` is a composite action consumer repos call to run `deploy.py` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository. It doesn't check anything out itself - `target-manifest` is read from whatever the caller's own preceding `actions/checkout` step already put on disk, and `deploy.py` itself comes along automatically via `$GITHUB_ACTION_PATH`. `.github/actions/renovate/` is the same shape, scoped to `renovate.py` instead. The `releases/{timestamp}`/`current` symlink pattern exists for atomicity, not for rollback: a deploy either fully lands and only then switches the symlink as its last step, or fails partway and leaves `current` untouched — never a partially-applied app. Rendered config templates are part of this same guarantee - they're written into the release tree and travel inside the release tarball, not pushed separately or in place. There is deliberately no automated rollback, and manual rollback (point `current` at an old release directory by hand) is not a supported/maintained path — it wouldn't restore a floating-tag image's historical version, and in practice fixing forward through the normal deploy path is simpler and safer than reasoning about what a partial rollback actually restores. @@ -409,10 +409,10 @@ Each app in a target's `apps` mapping may list its own `env_refs` — release re - **traefik**: Entry point, uses external network. Configured entirely via `TRAEFIK_*` env vars, no config file - see "Traefik Integration" above. - **cloudflared**: Optional Cloudflare Tunnel client, joins only the `traefik` network - see "Traefik Integration" above. - Apps with databases include a versioned template (e.g. `postgres-18.yml`) and create app-specific database named `${APP_NAME}` -- Config templates use `envsubst`-equivalent substitution (`deploy/render.py`) - variables must be shell-compatible (`${VAR}` syntax) +- Config templates use `envsubst`-equivalent substitution (`.github/actions/deploy/render.py`) - variables must be shell-compatible (`${VAR}` syntax) Watchtower was dropped entirely (there is no `apps/watchtower/`, and no app -carries its label) — `deploy/deploy.py` already runs `docker compose pull && +carries its label) — `.github/actions/deploy/deploy.py` already runs `docker compose pull && up -d` for every app in a target's `apps` mapping on every deploy, which made Watchtower's own polling redundant. See `RETIRED.md`. diff --git a/README.md b/README.md index 7c5e2ee..6c13b19 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ A target server needs only: - **Docker** >= 20.10 - **Docker Compose** >= 2.0 -- SSH access for the deploy key configured in that target's `credentials` +- SSH access for the deploy key configured in that target's `ssh_private_key_secret` ## Automated Deploy -Deployment goes through [`deploy-shared.yml`](.github/workflows/deploy-shared.yml) (documented in the GitHub Actions section below), a reusable workflow wrapping [`deploy/deploy.py`](deploy/deploy.py) behind plain deploy vocabulary — `hosts`, `app-refs`, `apps`. +Deployment goes through the [`deploy`](.github/actions/deploy) composite action (documented in the GitHub Actions section below) wrapping [`deploy.py`](.github/actions/deploy/deploy.py) behind one input — `target-manifest`, a path to that target's own manifest file, which the action reads itself rather than receiving `hosts`/`app_refs`/`apps` already flattened. It runs as a step inside a job the caller already checked its own repository out for, so the action needs no checkout logic of its own. The deploy is push-based and runs entirely on the GitHub Actions runner: @@ -58,21 +58,24 @@ flightdeck/ │ ├── postgres/ # PostgreSQL data │ └── {app-name}/ # Each app's data that must survive across releases │ -├── deploy/ -│ ├── deploy.py # Push-based deploy entrypoint (runs on the CI runner) -│ ├── resolve.py # owner/repo@tag[:asset] release ref resolution/download -│ ├── collisions.py # Ciphertext-based env key collision detection -│ ├── vault.py # SOPS decryption -│ └── render.py # envsubst-equivalent config template rendering ├── .github/ │ ├── actions/ │ │ ├── build-bundle/ # Build and upload a zip bundle from given paths │ │ ├── build-apps-bundle/ # Build and upload an apps/ catalog bundle │ │ ├── encrypt-env/ # Encrypt a target env and upload it to a release -│ │ └── load-yaml-matrix/ # Read a directory of YAML manifests into a workflow matrix +│ │ ├── load-vaults-matrix/ # Read vaults/ into a workflow matrix, minimal shape validation +│ │ ├── load-targets-matrix/ # Same, but for targets/ specifically - validates the required shape +│ │ ├── deploy/ # Push-based deploy against a target manifest (deploy.py + friends) +│ │ │ ├── deploy.py # Deploy entrypoint (runs on the CI runner) +│ │ │ ├── resolve.py # owner/repo@tag[:asset] release ref resolution/download +│ │ │ ├── collisions.py # Ciphertext-based env key collision detection +│ │ │ ├── vault.py # SOPS decryption +│ │ │ └── render.py # envsubst-equivalent config template rendering +│ │ └── renovate/ # Re-pull/recreate one-or-more apps' containers (renovate.py) │ └── workflows/ -│ ├── deploy-shared.yml # Reusable deployment workflow -│ └── release.yml # Release Please + publish Flightdeck assets +│ ├── deploy.yml # Manual redeploy of every target +│ ├── renovate.yml # Manual renovate of one-or-more apps across every matching target +│ └── release.yml # Release Please + publish Flightdeck assets + auto-deploy │ ├── vaults/ # Encrypted env asset configurations, one per app └── targets/ # Deployment targets @@ -109,7 +112,7 @@ Variable names inside a compose file are always bare, never prefixed with the ap - **internal** - Isolated network for app-to-app communication - **databases** - Dedicated network for database services (PostgreSQL, Redis, MongoDB) -`traefik` and `databases` are created on the target host by `deploy/deploy.py` (derived from `apps/networks.yml`'s `external: true` entries); `internal` is created by Docker Compose itself. +`traefik` and `databases` are created on the target host by [`deploy.py`](.github/actions/deploy/deploy.py) (derived from `apps/networks.yml`'s `external: true` entries); `internal` is created by Docker Compose itself. ## Adding a New Application @@ -184,7 +187,9 @@ Useful as a source of ready-made Docker Compose definitions when adding a new ap ## GitHub Actions -This repository provides four composite actions under `.github/actions/` (`build-bundle`, `build-apps-bundle`, `encrypt-env`, and `load-yaml-matrix`) and one reusable workflow, `deploy-shared.yml`. +This repository provides seven composite actions under `.github/actions/` (`build-bundle`, `build-apps-bundle`, `encrypt-env`, `load-vaults-matrix`, `load-targets-matrix`, `deploy`, and `renovate`). A reusable workflow only pays for itself when the reused thing genuinely needs its own multiple jobs or job-level config (`permissions`, `concurrency`, etc.); `deploy`/`renovate` are each a single job's worth of steps, and a composite action gets that for free without a `workflow_call` boundary's checkout/secrets ceremony (see [`deploy`](#deploy) below for what that ceremony would otherwise cost). + +The one exception is `deploy.yml` itself: it also declares `workflow_call`, purely so `release.yml` can call it directly instead of duplicating its `load-targets`+`deploy` job pair — see "`deploy.yml`" below. --- @@ -231,20 +236,17 @@ hosts: - deploy@app1.example.com - deploy@app2.example.com path: ~/flightdeck # optional, default shown -credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET - sops_age_key: MAINFRAME_AGE_PRIVATE_KEY +ssh_private_key_secret: DEPLOY_SSH_PRIVATE_KEY +sops_age_key_secret: MAINFRAME_AGE_PRIVATE_KEY ``` -Credential fields contain GitHub Variable/Secret names, never credential values. `app_refs` and `hosts` are YAML arrays; `apps` is a mapping from app name to that app's own `env_refs` array. Each host uses the SSH `user@host` format. `app_refs` must list at least one app bundle — flightdeck's own `apps/` catalog is just another entry, not implicit. `env_refs` is optional — omit it (or leave it `[]`) for an app that genuinely needs zero vault-sourced values (e.g. `beszel` above); it still gets a `.env` with `APP_NAME`/`DATA_DIR`, just no vault is fetched or decrypted for it. Don't create a vault manifest with an empty `env:` just to satisfy this field - there's nothing to encrypt, so there's nothing to gain from one. When `env_refs` is given, it must be non-empty; `deploy/deploy.py` decrypts and concatenates all of an app's sources into that app's own `.env` on the runner, failing loud on any key collision — but only within that one app's own sources. Two different apps' vaults sharing a key (e.g. both declaring `DOMAIN`) is expected, since each app gets a separate `.env`. `credentials.secrets.sops_age_key` names the GitHub Secret holding this target's *private* age key — the one used to decrypt its vaults, matching the public key in `keys/.pub` used to encrypt them. +`ssh_private_key_secret`/`sops_age_key_secret` are GitHub Secret *names*, never the credential values themselves - the `_secret` suffix says so explicitly, since a flat field like `sops_age_key` could otherwise read as the key material itself. `sops_age_key_secret` names the GitHub Secret holding this target's *private* age key - the one used to decrypt its vaults, matching the public key in `keys/.pub` used to encrypt them. Tailscale credentials live outside the target manifest entirely (`vars.TAILSCALE_OAUTH_CLIENT_ID`/`secrets.TAILSCALE_OAUTH_SECRET`, referenced directly by the workflows below) since the tailnet is shared infrastructure, not something that varies per target. + +`app_refs` and `hosts` are YAML arrays; `apps` is a mapping from app name to that app's own `env_refs` array. Each host uses the SSH `user@host` format. `app_refs` must list at least one app bundle — flightdeck's own `apps/` catalog is just another entry, not implicit. `env_refs` is optional — omit it (or leave it `[]`) for an app that genuinely needs zero vault-sourced values (e.g. `beszel` above); it still gets a `.env` with `APP_NAME`/`DATA_DIR`, just no vault is fetched or decrypted for it. Don't create a vault manifest with an empty `env:` just to satisfy this field - there's nothing to encrypt, so there's nothing to gain from one. When `env_refs` is given, it must be non-empty; [`deploy.py`](.github/actions/deploy/deploy.py) decrypts and concatenates all of an app's sources into that app's own `.env` on the runner, failing loud on any key collision — but only within that one app's own sources. Two different apps' vaults sharing a key (e.g. both declaring `DOMAIN`) is expected, since each app gets a separate `.env`. A vault manifest's `env:` value is either `${NAME}` (a reference — look up the GitHub Secret/Variable named `NAME`) or a bare literal (any other value, used as-is with no lookup at all — see `DISABLE_SIGNUP: true` above). Use a literal for a value that's fixed for this target but isn't a secret and doesn't need a GitHub Secret/Variable to exist just to hold it. -`load-yaml-matrix` reads every file in `vaults/` or `targets/` into a matrix — it does not validate the manifest shape. Each manifest's fields are the responsibility of whatever consumes them: `encrypt-env` re-parses and validates its own manifest from `manifest`, and the workflows calling `deploy-shared.yml` apply `path`/`keep-releases` defaults and pull `credentials.secrets`/`credentials.variables` values directly from the matrix item. +[`load-vaults-matrix`](.github/actions/load-vaults-matrix) reads every file in `vaults/` into a matrix, checking only that `asset`/`keys`/`env` are present and non-empty — `encrypt-env` still re-parses and fully validates its own manifest from `manifest` (field-level shape: name patterns, source references, and so on; see "`encrypt-env`" below). Targets go through the more specific [`load-targets-matrix`](.github/actions/load-targets-matrix) instead, which validates the shape above (`hosts`, `app_refs`, `apps`, `ssh_private_key_secret`, `sops_age_key_secret` all required) the same way. Either way, a structurally broken manifest fails here, before it ever reaches a checkout+dependency-install on a different job entirely. The workflows calling the [`deploy`](#deploy)/[`renovate`](#renovate) actions then pull `matrix.ssh_private_key_secret`/`matrix.sops_age_key_secret` directly, to resolve actual secret values by name. --- @@ -330,37 +332,104 @@ Requires `contents: write` permission on the calling job. `flightdeck-apps.zip` --- -### `deploy-shared.yml` +### `deploy` + +Runs [`deploy.py`](.github/actions/deploy/deploy.py) against a target manifest already present in the caller's own checkout. Intended to be used from a private consumer repository that owns both the config and secrets side (a `targets/*.yml` manifest shaped like the one above, the SSH key, encrypted `.sops.env` releases, the age private key, etc.) — this repository does not hold any deploy secrets itself. That manifest's `apps..env_refs` entries typically reference that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. + +This used to be a `workflow_call` reusable workflow (`deploy-shared.yml`), which needed its own second `checkout` (`repository: job.workflow_repository, ref: job.workflow_sha`) just to get *this* repository's own `deploy.py` onto the runner, and a `target-manifest-ref` input to keep that checkout's ref in sync with whatever ref the caller had already resolved its matrix from. As a composite action, neither problem exists: `deploy.py` comes along automatically via `$GITHUB_ACTION_PATH` whenever this is referenced as `owner/repo/.github/actions/deploy@ref`, and `target-manifest` is just read from whatever the caller's own preceding `actions/checkout` step already put on disk — no second checkout, no ref to keep in sync, because there's only ever one checkout to begin with. -Runs [`deploy/deploy.py`](deploy/deploy.py) from this repository against the caller-supplied hosts. Intended to be called from a private consumer repository that owns both the config and secrets side (SSH key, encrypted `.sops.env` releases, the age private key, etc.) — this repository does not hold any deploy secrets itself. `apps..env_refs` entries typically reference that same calling repository via `${{ github.repository }}`, since it's both the config and secrets source. +The interface is a single path, not flattened deploy vocabulary — the caller never re-serializes its target's `hosts`/`app_refs`/`apps`/`path` through `toJson(...)`, and `deploy.py` never receives them as separate fields; it parses and validates the manifest itself. `sops-age-key`/`ssh-private-key` are the two things that genuinely can't live in that file - secret *values*, resolved by the caller from the manifest's own `sops_age_key_secret`/`ssh_private_key_secret` fields (GitHub Secret *names*) and passed in directly. The action then resolves and downloads every ref, decrypts and renders each app's env and config, merges the release, and pushes the finished result to each host over SSH — see "Automated Deploy" above for the full sequence. -The interface is plain deploy vocabulary — callers never see `deploy.py`'s internals or hand-write its JSON config; the workflow builds that internally and pipes it to `python3 deploy/deploy.py` on stdin. The runner resolves and downloads every ref, decrypts and renders each app's env and config, merges the release, and pushes the finished result to each host over SSH — see "Automated Deploy" above for the full sequence. +Tailscale is optional, not a dependency of this action: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the hosts, or reaches them some other way. -Tailscale is optional, not a dependency of this workflow: set `tailscale-oauth-client-id` (and the matching `tailscale-oauth-secret`) to have the runner join a tailnet as an ephemeral node before deploying. Leave both unset to skip that step entirely — e.g. when the job already runs on a self-hosted runner with network access to the hosts, or reaches them some other way. +Like [`renovate`](#renovate), this action has no notification logic of its own - unlike `renovate`, a deploy doesn't need a "did anything actually change" check to decide whether to notify: reaching this point at all already means a real deploy just landed, so the caller's own following step notifies unconditionally (it simply never runs if the `deploy` step above it failed, same as any other step in a job). ```yaml jobs: deploy: - uses: rubykatzen/flightdeck/.github/workflows/deploy-shared.yml@v0.11.1 - with: - hosts: '["deploy@app1.example.com", "deploy@app2.example.com"]' # required JSON array - app-refs: '["rubykatzen/flightdeck@latest"]' # required non-empty JSON array - apps: '{"traefik": {"env_refs": ["${{ github.repository }}@latest:mainframe-traefik.sops.env"]}}' # required non-empty JSON object - # path: ~/flightdeck # optional, default shown - # keep-releases: 5 # optional, default shown - tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) - tailscale-tags: tag:ci # default: tag:ci - secrets: - ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} - sops-age-key: ${{ secrets.MAINFRAME_AGE_PRIVATE_KEY }} - tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # optional, required only if tailscale-oauth-client-id is set + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: rubykatzen/flightdeck/.github/actions/deploy@v0.11.1 + with: + target-manifest: targets/mainframe.yml # required, path in this repository + ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + sops-age-key: ${{ secrets.MAINFRAME_AGE_PRIVATE_KEY }} + github-token: ${{ secrets.GITHUB_TOKEN }} + # tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} # optional, default: unset (skip joining a tailnet) + # tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} # required only if tailscale-oauth-client-id is set + # tailscale-tags: tag:ci # default: tag:ci + - name: Notify Telegram + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Deploy: mainframe updated" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} ``` -The `@v0.11.1` pin on the `uses:` line only controls which ref runs `deploy/deploy.py` itself. `app-refs` entries are separate and don't have to match the workflow pin. +The `@v0.11.1` pin on the `uses:` line only controls which ref this action's own code runs at. `target-manifest`'s own `app_refs` entries are separate and don't have to match it. + +#### `deploy.yml` + +This repository's own `deploy.yml` is the manual-redeploy entry point shown above (`workflow_dispatch`, no inputs — it always redeploys every target in `targets/`), but it also declares `on.workflow_call` with no inputs of its own, purely so `release.yml` can call it directly instead of duplicating its `load-targets`+`deploy` job pair: + +```yaml +jobs: + deploy: + needs: [release, upload-apps, encrypt] + if: needs.release.outputs.release_created == 'true' + uses: $/.github/workflows/deploy.yml + secrets: inherit +``` + +`secrets: inherit` is safe here specifically because both workflows live in this same repository — `deploy.yml` already has native access to every one of this repository's own secrets when triggered directly, so inheriting them from `release.yml` (also this repository) doesn't actually broaden anything. This is the one narrow case where `secrets: inherit`'s same-organization restriction (see [`renovate`](#renovate) below) isn't a concern at all, since there's no organization boundary being crossed in the first place. The trade-off: `release.yml`'s deploy no longer includes the release tag in its Telegram message, since `deploy.yml`'s own generic "`{target}` updated" text doesn't know it - not worth reintroducing an input and a conditional `format(...)` expression just to preserve that one detail. + +--- + +### `renovate` + +Re-pulls and recreates one-or-more apps' containers on a target already present in the caller's own checkout, without touching versions — no new app bundle, no new vault-sourced env, no rebuilt release tree. Just `docker compose pull && docker compose up -d` per requested app, against its already-current release, on each of the target's hosts. Sibling to [`deploy`](#deploy), same checkout-free shape, deliberately narrower job. + +`apps` is a JSON array, so one run can renovate several apps at once (e.g. `["traefik","rybbit"]`) — pass a single-element array for the one-app case, or an empty array for every app the target runs. Not every target runs every requested app; [`renovate.py`](.github/actions/renovate/renovate.py) decides that itself from the target manifest's own `apps` mapping and simply does nothing — never opening an SSH connection — if none of the requested apps are present there. + +It never touches `app_refs`/`env_refs`, never re-decrypts a vault, never rebuilds the release tree; it just picks up a new image behind an existing tag. To tell whether a host's image actually changed (rather than the pull being a no-op), it compares `docker compose images -q` output before and after the pull, and exposes `updated`/`updated-hosts`/`target-name` (derived from the manifest's own filename) as action outputs - `updated-hosts` lists `app@host` pairs, since more than one app may have been renovated in the same run. + +Unlike `deploy`, this action has no notification logic of its own — it just reports whether anything changed. `renovate.yml` below is what actually decides to notify, as its own separate step reading this action's outputs; a different caller is free to wire up a different channel, or none at all, without forking this action. + +```yaml +jobs: + renovate: + needs: load-targets + if: needs.load-targets.outputs.count != '0' + strategy: + matrix: ${{ fromJson(needs.load-targets.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: rubykatzen/flightdeck/.github/actions/renovate@v0.11.0 + id: renovate + with: + apps: ${{ inputs.apps || '[]' }} # JSON array, e.g. '["traefik","rybbit"]'; '[]' on the nightly cron run + target-manifest: ${{ matrix.manifest }} + ssh-private-key: ${{ secrets[matrix.ssh_private_key_secret] }} + # tailscale-oauth-client-id: ${{ vars.TAILSCALE_OAUTH_CLIENT_ID }} + # tailscale-oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} + - name: Notify Telegram + if: steps.renovate.outputs.updated == 'true' + uses: rubykatzen/baseline/.github/actions/send-telegram-message@v0.16.1 + with: + message: "Renovate: updated on ${{ steps.renovate.outputs.target-name }} (${{ steps.renovate.outputs.updated-hosts }})" + telegram-bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + telegram-chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} +``` + +`renovate.yml` also runs on a nightly `schedule` (`0 3 * * *`), with `apps` defaulting to `[]` — a cron trigger can't supply `workflow_dispatch` inputs at all, so the empty-array-means-everything behavior above exists specifically to give the scheduled run something to pass. + +**Known gap:** if none of the requested `apps` match any target at all (a typo, say), every matrix job just does nothing and the whole run still reports success — there's no cheap way to fail loudly on "zero matches across the board" without a job that waits on the whole matrix and inspects its results. ## License diff --git a/targets/heimdall.yml b/targets/heimdall.yml index ab2a7a2..47cb477 100644 --- a/targets/heimdall.yml +++ b/targets/heimdall.yml @@ -12,10 +12,5 @@ apps: - rubykatzen/flightdeck@latest:cloudflared.sops.env hosts: - root@100.75.53.13 -credentials: - variables: - tailscale_oauth_client_id: TAILSCALE_OAUTH_CLIENT_ID - secrets: - ssh_private_key: DEPLOY_SSH_PRIVATE_KEY - tailscale_oauth_secret: TAILSCALE_OAUTH_SECRET - sops_age_key: HEIMDALL_AGE_PRIVATE_KEY +ssh_private_key_secret: DEPLOY_SSH_PRIVATE_KEY +sops_age_key_secret: HEIMDALL_AGE_PRIVATE_KEY