diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 8e3fdfc..d6fc825 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -40,7 +40,7 @@ jobs: validate: name: Validate sources job needs: [ assert-ref ] - uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@cd7bae9ea830d2e42d1f7bb6e45b3e7f631c0668 # 2.0.448 + uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 permissions: contents: read with: @@ -57,7 +57,7 @@ jobs: deploy: name: Deploy site job needs: [ validate ] - uses: ptr727/ProjectTemplate/.github/workflows/deploy-site-task.yml@cd7bae9ea830d2e42d1f7bb6e45b3e7f631c0668 # 2.0.448 + uses: ptr727/ProjectTemplate/.github/workflows/deploy-site-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 with: environment: ${{ inputs.environment }} permissions: diff --git a/.github/workflows/merge-bot-pull-request.yml b/.github/workflows/merge-bot-pull-request.yml index b7712d6..8b842ff 100644 --- a/.github/workflows/merge-bot-pull-request.yml +++ b/.github/workflows/merge-bot-pull-request.yml @@ -19,7 +19,7 @@ jobs: merge-bot: name: Merge bot pull request job - uses: ptr727/ProjectTemplate/.github/workflows/merge-bot-task.yml@37aa042042f51655ce368ec62dd1a655ac4b3716 # 2.0.428 + uses: ptr727/ProjectTemplate/.github/workflows/merge-bot-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 secrets: CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }} CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index ad4fa90..2c339f8 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -17,7 +17,7 @@ jobs: # Also replaces this repo's own dispatch-ref assertion: the hub task errors identically on a ref other than main/develop. plan: name: Plan release job - uses: ptr727/ProjectTemplate/.github/workflows/publish-plan-task.yml@37aa042042f51655ce368ec62dd1a655ac4b3716 # 2.0.428 + uses: ptr727/ProjectTemplate/.github/workflows/publish-plan-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 with: event_name: ${{ github.event_name }} actor: ${{ github.actor }} @@ -31,7 +31,7 @@ jobs: name: Validate sources job needs: [plan] if: ${{ needs.plan.outputs.publish == 'true' }} - uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@cd7bae9ea830d2e42d1f7bb6e45b3e7f631c0668 # 2.0.448 + uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 permissions: contents: read with: @@ -52,7 +52,7 @@ jobs: name: Publish project release job needs: [plan, validate] if: ${{ needs.plan.outputs.publish == 'true' }} - uses: ptr727/ProjectTemplate/.github/workflows/build-release-task.yml@37aa042042f51655ce368ec62dd1a655ac4b3716 # 2.0.428 + uses: ptr727/ProjectTemplate/.github/workflows/build-release-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 permissions: contents: write actions: write diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml index f8e49a5..a8d4f6d 100644 --- a/.github/workflows/test-pull-request.yml +++ b/.github/workflows/test-pull-request.yml @@ -19,7 +19,7 @@ jobs: validate: name: Validate sources job - uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@cd7bae9ea830d2e42d1f7bb6e45b3e7f631c0668 # 2.0.448 + uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@be4d3bf0a6c87adebe3aa27d32a57291c8a919b9 # 2.0.453 permissions: contents: read with: diff --git a/.secrets/README.md b/.secrets/README.md index e97323e..f2d74f5 100644 --- a/.secrets/README.md +++ b/.secrets/README.md @@ -9,10 +9,10 @@ exposing one. The documented local convention is `~/.secrets/`, not this directory. CI supplies the same values directly from the GitHub Environment instead, reading no file here at all. -`ENV_FILE= deploy/make-release.sh` and `ops/install.sh` both resolve a relative `ENV_FILE` -against `$HOME/.secrets`, refuse a traversing one, and default to -`~/.secrets/Blog.local.production.env`. An absolute `ENV_FILE` is honored as given rather than -resolved against `~/.secrets/`, an escape hatch rather than the documented shape. +`ENV_FILE= deploy/make-release.sh` resolves a relative `ENV_FILE` against `$HOME/.secrets`, +refuses a traversing one, and defaults to `~/.secrets/Blog.local.production.env`. An absolute +`ENV_FILE` is honored as given rather than resolved against `~/.secrets/`, an escape hatch rather +than the documented shape. `~/.secrets/` is shared across every repo on the host, so each of this repo's files carries the `Blog.` prefix: diff --git a/.secrets/example.env b/.secrets/example.env index 0d81837..2df85d1 100644 --- a/.secrets/example.env +++ b/.secrets/example.env @@ -4,15 +4,9 @@ # This file states the shape and a placeholder. Add a value here and describe it there, # or checks/check-env-docs.py fails. # -# It fills two destinations, marked below, because a value belongs to whichever machine -# holds it. Copy the section you need rather than the whole file. -# -# ~/.secrets/Blog...env on a workstation, one file per environment -# /etc/vps-backup-pull.env on the backup host, or let ops/install.sh write it -# -# A value appearing in both sections is spelled the same way in both, deliberately. One -# name per thing means the side that writes and the side that reads cannot disagree, which -# is also why ops/install.sh copies values across rather than translating them. +# One destination: ~/.secrets/Blog...env on a workstation, one file +# per environment. The off-host VPS backup pull, and the environment file it reads, are +# maintained outside this repository -- see OPERATIONS.md, Working With the VPS. # # Naming convention: the prefix names whatever owns the value, not whatever reads it. # SITE_ is the generic deploy contract, DEPLOY_ is the release tooling, CADDY_ is the @@ -87,39 +81,7 @@ VPS_TRAEFIK_LOG=/var/log/traefik/access.log # The two agent channel files on the VPS. VPS_COMMS_DIR=/srv/agent-comms -# ============================================================================= -# Both destinations -# ============================================================================= -# These name the off-host copy, so the pull writes them and the log review reads them. - -# Off-host archives and the plaintext hostconfig tree beside them. -BACKUP_ARCHIVE_ROOT=/path/to/backup/vps - -# Off-host copy of the rotated logs. Mode 700, since query strings are logged in full. +# Off-host copy of the rotated logs, written by the pull maintained outside this +# repository. Read by the log review. Mode 700 on the backup host, since query strings +# are logged in full. LOG_ARCHIVE_ROOT=/path/to/backup/vps-logs - -# The rotated access logs on the VPS, and the source of that copy. -VPS_TRAEFIK_LOG_ARCHIVE=/var/log/traefik/archive - -# ============================================================================= -# /etc/vps-backup-pull.env -# ============================================================================= -# On the backup host. Also needs VPS_SSH_HOST and the three values above. -# -# systemd parses this file itself rather than passing it to a shell, so there is no -# expansion and no command substitution: a $ or a backtick is a literal character. It does -# strip matching quotes, which is why a value containing spaces is quoted and arrives -# without them. -# -# VPS_SSH_HOST, BACKUP_ARCHIVE_ROOT and LOG_ARCHIVE_ROOT have no defaults in the pull. An -# address and a destination belong to one host, and a wrong-but-valid destination is a -# backup nobody can find, so it names what is missing and refuses to run. - -# The layout on the VPS, the same for any host running this stack. -VPS_ARCHIVE_DIR=/var/backups/pangolin -VPS_BLOG_LOG_DIR=/var/log/blog/legacy - -# Key auth only, since the VPS has password auth disabled. -# Quoted because it contains spaces: this file is sourced by a shell for the secrets/ -# half, where a bare value would run everything after the first space as a command. -SSH_OPTS="-o ConnectTimeout=15 -o BatchMode=yes" diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index b5b126b..ec62565 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -33,31 +33,11 @@ Held in `~/.secrets/Blog...env`, one file per environment, | `CAPTURE_AUTHOR_SLUG` | the old platform's author slug, used to backfill the author archive and its pagination | Optional, and an account name rather than a site value, which is why it is a variable at all. Unset, [`capture/classify.py`](./capture/classify.py) skips the backfill and says so, rather than emitting a list that is silently short by the author URLs. Environment-independent. | | `VPS_SSH_HOST` | the VPS administrative login | Not the deploy account. See "Two credentials" below. Environment-independent. | | `VPS_TRAEFIK_LOG` | today's live access log on the VPS, still being appended to | Never pulled, because rotation is what makes a file eligible. An analysis covering today reads it over SSH. Nothing sources it. | -| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated access logs on the VPS, and the source of the off-host copy | Also read by the pull, below. | | `VPS_COMMS_DIR` | the two agent channel files on the VPS | Nothing sources it, and the transfer commands in `OPERATIONS.md` are spelled out rather than using it. See "The one place indirection is wrong" below. | -| `BACKUP_ARCHIVE_ROOT` | the off-host encrypted archives and the plaintext `hostconfig` tree beside them | Written by the pull, read by a rebuild. | -| `LOG_ARCHIVE_ROOT` | the off-host copy of the rotated logs | Written by the pull, read by the log review. | +| `LOG_ARCHIVE_ROOT` | the off-host copy of the rotated logs | Written by a pull maintained outside this repository, read by the log review. | Three more are named in the template but commented out, because CI resolves them from the GitHub Environment and a local run deploys to a path and needs none of them: `DEPLOY_SSH_HOST`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_KNOWN_HOSTS`. They are listed there so the local file and the environment describe the same shape. -## The backup host - -Held in `/etc/vps-backup-pull.env`, read by `vps-backup-pull` through the unit's `EnvironmentFile`. Template: [`.secrets/example.env`](./.secrets/example.env). [`ops/install.sh`](./ops/install.sh) generates it by copying from the repository environment file, which is why the four shared names are spelled identically in both. - -| Value | Names | Notes | -| --- | --- | --- | -| `VPS_SSH_HOST` | where to pull from | Required. No default. | -| `BACKUP_ARCHIVE_ROOT` | where the archives and host config land | Required. No default. | -| `LOG_ARCHIVE_ROOT` | where both log sets land | Required unless `--no-logs`. No default. Mode 700, because query strings are logged in full. | -| `VPS_ARCHIVE_DIR` | the encrypted archives on the VPS | Defaults to the documented layout. | -| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated edge access logs on the VPS | Defaults to the documented layout. | -| `VPS_BLOG_LOG_DIR` | one-off Caddy container dumps on the VPS, kept from before rotation existed | Defaults to the documented layout. | -| `SSH_OPTS` | the SSH options the transfer uses | `BatchMode` makes an unusable key fail immediately rather than hanging a timed run on a password prompt nobody sees. | - -**The three marked required carry no default on purpose.** An address and a destination belong to one host, and a wrong-but-valid destination is a backup nobody can find, so the pull names what is missing and refuses to run rather than falling back to something plausible. - -**`systemd` parses this file itself rather than passing it to a shell**, so there is no expansion and no command substitution, and a `$` or a backtick is a literal character. It does strip matching quotes, verified rather than assumed, so a value containing spaces is quoted and arrives without them. That matters because [`.secrets/example.env`](./.secrets/example.env) is also sourced by a shell for the other destination, where an unquoted value would run everything after the first space as a command. - ## The GitHub Environments Held on the `production` and `staging` environments. The deploy workflow reads no file. diff --git a/OPERATIONS.md b/OPERATIONS.md index f59fa7e..8fc115c 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -210,28 +210,22 @@ ssh "$VPS_SSH_HOST" true && echo reachable | --- | --- | --- | | `VPS_SSH_HOST` | the administrative login | the VPS | | `VPS_TRAEFIK_LOG` | today's live access log, still being appended to | the VPS | -| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated access logs, and the source of the off-host copy | the VPS | | `VPS_COMMS_DIR` | the two agent channel files | the VPS | | `LOG_ARCHIVE_ROOT` | the off-host copy of the rotated logs | the backup host | -| `BACKUP_ARCHIVE_ROOT` | the off-host encrypted archives and the plaintext hostconfig tree beside them | the backup host | **There are two credentials to this host and picking the wrong one is the first mistake to avoid.** `DEPLOY_SSH_USER`, held per environment and used only by the deploy, reaches a confined account behind an `rrsync` forced command that can write one release tree and read nothing else. `VPS_SSH_HOST` is the ordinary administrative login used for everything on this page. They are deliberately separate credentials with different blast radii, so reaching for the deploy account to read a log fails in a way that reads like an outage, and reaching for the admin account to deploy grants far more than the deploy needs. -**The off-host copy is made by a script in this repository, [`ops/vps-backup-pull`](./ops/vps-backup-pull), on a `systemd` timer on the backup host.** It copies three things off the VPS into `BACKUP_ARCHIVE_ROOT` and `LOG_ARCHIVE_ROOT`: the encrypted archives, a plaintext copy of the same non-secret host files, and the rotated access logs. What it does, the three behaviors that look like bugs and are not, how to install it, and how to check it ran are in [`ops/README.md`](./ops/README.md). Read the unit and its last run on the backup host rather than trusting a schedule written down anywhere, including here. +**The off-host copy, maintained outside this repository, writes the rotated access logs to `LOG_ARCHIVE_ROOT`, and this section covers nothing more about it.** Its installation, how the VPS itself is provisioned, and its trust model are the backup host's own configuration to document, not this repository's. What "Log Review" needs from its schedule and copy behavior, to read the logs correctly, is covered there instead. Read the unit and its last run on the backup host rather than trusting a schedule written down anywhere, including here. -**It is a pull rather than a push, and nothing on the VPS knows it happens.** That direction is the security property rather than an implementation detail: the backup host holds a key the VPS trusts, and the VPS holds no credential reaching any other system, so a compromise of the web server cannot walk into the backups that exist to survive it. - -**Both sides use one set of names, so there is nothing to reconcile.** The pull writes `BACKUP_ARCHIVE_ROOT` and `LOG_ARCHIVE_ROOT` and the log review reads the same two, spelled the same way, and [`ops/install.sh`](./ops/install.sh) generates the pull's `EnvironmentFile` from this repository's `~/.secrets/` file by copying rather than translating. Every value is described once, in [`ENVIRONMENT.md`](./ENVIRONMENT.md), and [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if one is declared without a description or described without existing. +**`LOG_ARCHIVE_ROOT` is spelled the same way on both sides, so there is nothing to reconcile.** The pull writes it and the log review reads it, under the one name. Every value this repository reads or writes is described once, in [`ENVIRONMENT.md`](./ENVIRONMENT.md), and [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if one is declared without a description or described without existing. ```sh set -a; . ~/.secrets/Blog.local.production.env; set +a -ls -d "$LOG_ARCHIVE_ROOT" "$BACKUP_ARCHIVE_ROOT" +ls -d "$LOG_ARCHIVE_ROOT" ``` **Today's traffic is never in the off-host copy, and that is deliberate.** Rotation is what makes a file eligible to be pulled, so a live log would be copied as a torn prefix and fetched again on the next run. An analysis covering today therefore reads `VPS_TRAEFIK_LOG` over SSH and everything older from `LOG_ARCHIVE_ROOT`, and treats the two as one series joined on `StartUTC` rather than on which file a line came from. -**The plaintext `hostconfig` tree under `BACKUP_ARCHIVE_ROOT` is the readable copy of the VPS's own configuration**, carrying the same non-secret files the encrypted archives hold. It exists so a rebuild does not depend on the encryption key, which is not on the backup host and must never be put there, because beside the ciphertext it would make the encryption decorative. What that tree covers is whatever the VPS advertises, read from the host rather than duplicated here, so it tracks the host instead of drifting from a list. - **The channel transfers are the one exception, and they must stay literal.** The permission allowlist in `.claude/settings.local.json` matches the text of a command rather than what it expands to, so substituting `"$VPS_SSH_HOST:$VPS_COMMS_DIR/..."` into those two `rsync` lines turns an allowed command into one that prompts, while looking like a tidy-up that changed nothing. Use the values above everywhere else, and leave the two commands under "The Channel Between the Two Sides" spelled out exactly as they are written there. **What this section does not cover, and where it lives instead.** Reading the logs for content is "Log Review"; exchanging rounds with the agent that owns the host is "The Channel Between the Two Sides"; the boundary of which side fixes what is "Who Owns What"; and what a rebuild restores, including the host-key step that blocks both deploy and rollback, is "Backup and Restore". diff --git a/README.md b/README.md index b36127e..f5aacc5 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,6 @@ flowchart LR | [`checks/`][checks] | the URL contract and the gates that enforce it | | [`deploy/`][deploy] | the release script, the web-server config, and the redirect maps | | [`capture/`][capture] | the migration's provenance tooling, and how the site was derived from the old platform's exports | -| [`ops/`][ops] | the pull that copies the server's backups and access logs off it, and its schedule | | [`ENVIRONMENT.md`][environment] | every configuration value, described once | Every configuration value is described in [ENVIRONMENT.md][environment]. The deploy procedure and the server layout are in [OPERATIONS.md][operations]. @@ -231,7 +230,6 @@ Licensed under the [MIT License][license]\ [license]: ./LICENSE [migration-post]: ./content/posts/2026/08/01/moving-this-blog-from-wordpress-to-hugo.md [operations]: ./OPERATIONS.md -[ops]: ./ops/ [workflow]: ./WORKFLOW.md diff --git a/TODO.md b/TODO.md index 7c4607e..934a607 100644 --- a/TODO.md +++ b/TODO.md @@ -65,14 +65,14 @@ The site is built, gated in CI, and deployed to staging by pipeline. It is not y **Everything above ends; this section does not.** The list above finishes when the site serves its public address, and these two start there and never stop. Both are written down as procedures and neither has run on its cadence yet, which is the shape a recurring duty has right before it silently becomes a one-off. An end-to-end exercise of this system is a deploy **plus** the operations that follow it, so a run that proves staging and production and stops has proven the half that has an obvious finish line. - **Review the logs for non-200s**, daily for the first week after cutover, then monthly. Real traffic finds what the golden list missed and the crawl that produced the list cannot. Append anything new to `checks/golden-urls.txt` and add a redirect. Read the edge as well as Caddy: a request the proxy refused never reaches the site's log, so a count taken from Caddy alone is a floor, and a staging probe for `/wp-login.php` answered by the auth gate rather than by the site is the shape of what Caddy never sees. The procedure, the three tiers and what each is blind to, and the inward pass that names content nobody has ever requested are in [`OPERATIONS.md`](./OPERATIONS.md) "Log Review". -- **Pull the log off the VPS, on a schedule**, which is [#53][issue-53]. The access log is deliberately outside the nightly encrypted archives, because those are fourteen full copies with no dedupe and an append-only file would be multiplied by fourteen for no recovery benefit, so the VPS's 400-day window was the **only** copy until this ran. **It is installed**, as `vps-backup-pull.timer` at 09:00 UTC daily with `Persistent=true`, and a first copy exists: 42 archives and 4 log files, pulled 2026-08-08 12:59 UTC. **That run was started by hand, so the timer has never fired**, which is the distinction worth keeping until 2026-08-09 09:00 UTC proves the schedule rather than the script. One copy is a fact; "backed up daily" is still a unit file. +- **Pull the log off the VPS, on a schedule.** The access log is deliberately outside the nightly encrypted archives, because those are fourteen full copies with no dedupe and an append-only file would be multiplied by fourteen for no recovery benefit, so the VPS's 400-day window was the **only** copy until this ran. **It is installed**, as `vps-backup-pull.timer` at 09:00 UTC daily with `Persistent=true`, and a first copy exists: 42 archives and 4 log files, pulled 2026-08-08 12:59 UTC. **That run was started by hand, so the timer has never fired**, which is the distinction worth keeping until 2026-08-09 09:00 UTC proves the schedule rather than the script. One copy is a fact, and "backed up daily" is still a unit file. The pull itself, and the reconciliation this bullet's history refers to, moved out of this repository 2026-08-24, see "Open decisions" below -- kept here as the historical record of how it was proven to work. - **A rename on the VPS does not reach this copy, and nothing detects that it did not.** The pull deliberately passes no `--delete` for the logs, since that flag exists to mirror the VPS's fourteen-archive window and must never touch an append-only file. So when the host side renamed and merged its two mis-dated archives, the pre-fix name survived here: `access.log-2026-08-08`, 52 lines, every one of them 2026-08-07 traffic and every one already inside the merged `access.log-2026-08-07.gz`, which holds 58. Verified a strict subset with `comm -23` rather than assumed. **A line count over the off-host archive therefore returned 110 lines where 58 exist, half of them filed under a date whose traffic they are not** — which is exactly the defect the host side fixed, surviving on the copy the log review will read once the VPS's window rolls past what it needs. The general form is that any rename, merge, or re-compression of an already-pulled log leaves the old name here permanently. **That one file is deleted and the archive reads 58**, and the general case now has a mechanism: the host side keeps an append-only `RECONCILE.md` **inside the archive directory**, so the pull carries it alongside the data it explains rather than relying on a note in a channel file nobody rereads. It records what a file contained rather than what it was called, and it will be counted among the pulled log files. - **Read a date from a line's `StartUTC` rather than from the filename holding it.** That is the durable form of the lesson, and it is in [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" as well. - **An off-host copy also predates the timer.** The directory mtimes on the backup host are copy times where the file mtimes are the VPS's, and they show a pull at 2026-08-08 03:31 UTC that the service journal has no record of, because the script was run directly rather than through `systemd`. So a journal with one entry is not evidence of one copy. The whole set was audited both ways afterwards and nothing else had diverged: logs identical, 38 archives shared and identical in size, four newer on the VPS because they postdate the pull, four older retained off-host because the pull passes no `--delete`. - **Both halves were exercised 2026-08-08 between 16:18 and 16:22 UTC, deliberately without running the service.** `systemctl start` would have written the second journal entry that 2026-08-09 is supposed to prove, so the transport was exercised with `--dry-run` instead and the timer was read rather than triggered: `LAST` is `-` and the journal still holds exactly one entry, with `NEXT` inside the 15-minute randomized window after 09:00 UTC. **Read `NEXT` rather than remembering it**, because `systemctl enable` redraws that offset: it moved from 09:08:45 to 09:00:12 UTC when the unit was installed. **If a second entry exists before that time, someone ran it by hand and the schedule is still unproven.** The dry run reached the VPS over SSH and all three legs planned cleanly. Pending for the first timed run: five encrypted archives dated 2026-08-08, plus `RECONCILE.md` **and** `MANIFEST.txt`, so the log-file count rises by two non-log files rather than the one recorded above. No rotated access log is pending, which is correct, because the 00:00 UTC rotation that produces `access.log-2026-08-08` has not happened yet. - - **`--dry-run` named nothing, which made it a connectivity test wearing a preview's name.** `RSYNC_OPTS` carried only `-a --human-readable --info=stats1`, so a dry run printed transfer totals and not one filename, and "what will tomorrow's run bring" was unanswerable by the flag that exists to answer it. Fixed by adding `--itemize-changes` alongside `--dry-run`, which is how the pending set above was read. **The fix is committed at [`ops/vps-backup-pull`](./ops/vps-backup-pull) and installed 2026-08-08**, verified byte-identical to the committed copy. Installing it is a maintainer step, and it stacks with the unshipped change [#53][issue-53] already owes the VPS canonical at `/usr/local/share/pangolin-maint/vps-pull.sh`. - - **The VPS's older copy of the script is not a source, and reconciling the two is [#53][issue-53].** The committed copy carries the whole access-log leg, the `tell()` fix and the `VERIFIED` counter. The VPS's carries an install block and a no-sudo rationale that this side lacked, now folded in. Neither direction is a safe overwrite, so #53 is a merge rather than a copy, and copying the VPS's over the committed one would delete the log pull that [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" runs on. - - **The header's own install command pointed at `/usr/local/sbin/vps-backup-pull`, which nothing runs.** `vps-backup-pull.service` runs `/usr/local/bin/vps-backup-pull` and `/usr/local/sbin/` is empty, so following the instruction would have written a second copy nobody executes while `scp` and `chmod` both reported success and the timer went on running the old one. The canonical had already corrected this to a `sudo install` into `bin/`, and states the reason `bin/` is deliberate. **The stale block is replaced in the patch copy** with the canonical's wording plus an explicit refusal to run that `scp` until the divergence above is reconciled. It is another reason a plain overwrite in either direction is the wrong merge. + - **`--dry-run` named nothing, which made it a connectivity test wearing a preview's name.** `RSYNC_OPTS` carried only `-a --human-readable --info=stats1`, so a dry run printed transfer totals and not one filename, and "what will tomorrow's run bring" was unanswerable by the flag that exists to answer it. Fixed by adding `--itemize-changes` alongside `--dry-run`, which is how the pending set above was read. **The fix was committed to this repository's own copy of the script and installed 2026-08-08**, verified byte-identical to the committed copy at the time (the script itself has since moved, see "Open decisions" below). + - **The VPS's older copy of the script was not a source, and reconciling the two was [#53][issue-53].** That issue is no longer this repository's to track, since the canonical script is not committed here anymore. + - **The header's own install command pointed at `/usr/local/sbin/vps-backup-pull`, which nothing runs.** `vps-backup-pull.service` runs `/usr/local/bin/vps-backup-pull` and `/usr/local/sbin/` is empty, so following the instruction would have written a second copy nobody executes while `scp` and `chmod` both reported success and the timer went on running the old one. The canonical had already corrected this to a `sudo install` into `bin/`, and stated the reason `bin/` is deliberate. - **The outward pass ran end to end on 2026-08-08 traffic and found nothing to add.** 9,996 edge requests, 9,285 of them this repo's own deploy gate. Of the 711 that remain, 101 were site-host 404s across 73 distinct paths, and every one was a scanner shape. Only `/` and `/robots.txt` intersect the URL contract at all, and both are explained rather than open: `/robots.txt` 404ed until the 15:47:17 deploy and has answered 200 since 15:48:45, and `/` 404ed twice at the edge, below. **No legacy content URL 404ed, so `checks/golden-urls.txt` needs no addition from this run**, which is the expected result and is recorded because an unrecorded clean pass is indistinguishable from a pass nobody ran. - **Two site-host 404s came from the edge rather than the site, and belong to the VPS side.** `/` and `/favicon.ico` at 2026-08-08T15:39:01, one client, same second, both carrying no `ServiceName` at all where the other 99 carried `1-Blog-Production-service@http`. No router matched, so Traefik answered and the blog never saw the request, and requests to `/` seventeen seconds later were routed normally. It sits inside the window the host side was reconfiguring Pangolin in, which is a plausible cause and not a measured one. Raise it in the channel rather than diagnosing it from this side, and note that Caddy is structurally blind to it: a 404 count taken from the site's own log would report zero of these. @@ -93,11 +93,8 @@ The reference leaf the hub now ships carries one step this repo's deploy does no ## Open decisions -- **Resolved for the backup pull, 2026-08-08: it is in this repository at [`ops/`](./ops/).** The script, both `systemd` units, an `EnvironmentFile` template naming every path it uses, and a README covering what it does and how to check it. [`OPERATIONS.md`](./OPERATIONS.md) "Working With the VPS" names it and states which of its variables pair with which of this repo's. The reasoning below stands as the record of why, and the same question is still open for everything under it. **Installed 2026-08-08 with `ops/install.sh`**, which derives the address and both destinations from `~/.secrets/Blog.local.production.env`, and the account, the group and the mount from the invoking environment itself, so nothing is typed twice. Verified after the fact rather than from the installer's own output: `systemd` resolves `User=pieter`, `Group=users` and `RequiresMountsFor=/data/backup` from the drop-in, and the environment file is `600 root:root`. The running script is byte-identical to the committed one. Re-running the installer after the shell-gate reformat also exercised its idempotent path, which reported both config files already correct and replaced only the script, so a changed value is applied by running it again rather than by editing anything on the host. The root guard was exercised and refused. Installing did not spend the evidence for the first scheduled run, which then arrived: **the timer fired unattended on 2026-08-09 at 09:11:01 UTC**, with a second journal entry and `LAST` off `-`. Separately, [#53][issue-53] reconciles the VPS's older copy in both directions rather than by overwriting either. -- **Where the rest of the operational tooling lives, given that today it lives nowhere.** `vps-backup-pull`, its `systemd` units, and the environment variables naming both ends of the copy were an operational asset built from another agent's instructions, and they existed only on the Proxmox host. That host is the machine the backup runs *from*, so losing it loses both the copies and the means of making them, and the instructions that produced them are in a channel file this repository deliberately does not carry. Two candidate homes, and the choice is open: **here**, beside the deploy tooling the same host runs, or **the home-automation config repository**, with the rest of that host's configuration. The argument for the second is that nothing about the pull is specific to this site; the argument for the first is that [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" is the thing that stops working without it. - - **The pull itself is resolved and the reasoning is kept because it applies to everything still listed here.** What made it urgent was measured: the copy protected everywhere was the VPS's older one, while the copy that actually ran, carrying the log leg the review depends on, was in no snapshot and no repository. Committing it is what closed that, not the backup host's own off-site copy, which never reached the script. - - **The directory holding it is named as though it were disposable.** `~/vps-backup-pull-patch` reads as a patch staged against a source, and there is no source: it is the most complete copy of the script in existence. A directory named for a temporary artifact is the one a cleanup deletes, and nothing here would notice until a restore produced the wrong script. - - **The same reasoning points at the home-automation configuration repository for anything that is purely this host's**, since that is where the rest of the backup host's service configuration already lives. The pull is here instead because [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" is what stops working without it. Revisit if a second unrelated host service ends up here. +- **Resolved, and superseded again since: the backup pull and the VPS's cold-standup runbook moved out of this repository, 2026-08-24.** They now live in the maintainer's private infrastructure tooling, maintained outside this repository, alongside the rest of that host's own service configuration. `OPERATIONS.md` "Working With the VPS" documents only what this repository's own tooling still needs from that side: that `LOG_ARCHIVE_ROOT` exists and is where the pulled logs land. [#53][issue-53], reconciling the VPS's own older copy of the pull script against the committed one, is no longer this repository's to track, since neither copy is committed here anymore. +- **Where the rest of the operational tooling lives is no longer an open question.** `vps-backup-pull`, its `systemd` units, the VPS's cold-standup runbook, and a sanitized reference for the VPS's own current configuration all moved together, since nothing about any of it was specific to this site, which was the argument for moving it in the first place. This repository's own `OPERATIONS.md` and `ENVIRONMENT.md` were trimmed to describe only what this repository still needs, whether read by its tooling or relied on by a manual VPS procedure documented here. - **Resolved for the migration toolchain: the durable scripts are in this repository at [`capture/`](./capture/).** The fourteen split by whether they can ever run again rather than by whether they look reusable. A re-export re-runs the conversion chain, so `run-wp2hugo.sh`, `clean-content.py`, `restructure-content.py`, `localize-external.py` and `enumerate-media.py` are carried. `build-golden.py` and `classify.py` are carried as the record behind an append-only contract, and say so, since they cannot run once the old hosting ends. The crawl pair and the three fetchers are named in [`capture/README.md`](./capture/README.md) rather than carried, because their outputs are the durable artifact and the scripts hit a site that will be gone. - **The three copies of `build-redirects.py` are down to one.** The maintained copy moved from `checks/` to [`capture/`](./capture/build-redirects.py), which is where it belonged: it generates rather than gates, and `checks/` is defined as the contract and the gates that enforce it. Both stale copies in the capture are named by path in the capture README and marked stale, so finding one is not mistaken for finding a source. - **What `robots.txt` says, which is undecided and is the last non-mechanical item before M7b.** Recorded under "Next" above, where it blocks the cutover. diff --git a/ops/README.md b/ops/README.md deleted file mode 100644 index f192e23..0000000 --- a/ops/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Backup Pull - -The pull that copies the VPS's backup set and its access logs off the VPS. It runs on the **backup host**, not on the web server and not in CI, on a `systemd` timer. - -| File | Installs to | -| --- | --- | -| `install.sh` | nothing, it does the installing | -| `vps-backup-pull` | `/usr/local/bin/vps-backup-pull` | -| `vps-backup-pull.service` | `/etc/systemd/system/` | -| `vps-backup-pull.timer` | `/etc/systemd/system/` | -| `vps-backup-pull.service.d-local.conf.example` | `/etc/systemd/system/vps-backup-pull.service.d/local.conf` | -| [`example.env`](../.secrets/example.env) | `/etc/vps-backup-pull.env` | - -**The last two are required, not optional, and `install.sh` generates both.** Nothing in this directory names a machine, so the address, the destination paths, and the account are supplied at install time from values this repository already holds. A missing value stops the pull with the name of what is missing rather than falling back to something plausible, since a wrong-but-valid destination is a backup nobody can find. The two `.example` files document the format and are not the install path. - -## What it does - -Three legs, each skippable, in one direction only: - -| Leg | From the VPS | To the backup host | -| --- | --- | --- | -| archives | the encrypted backup set | `BACKUP_ARCHIVE_ROOT` | -| host config | the same non-secret files in plaintext | `BACKUP_ARCHIVE_ROOT/hostconfig` | -| access logs | the rotated edge logs | `LOG_ARCHIVE_ROOT` | - -**It is a pull rather than a push, and that is a security property rather than a convenience.** The backup host holds a key the VPS trusts, and the VPS holds no credential reaching anything else. A push would have to invert that, so a compromise of the web server would reach the backups that exist to survive it. - -**The plaintext host-config leg exists so a rebuild does not need the encryption key.** That key is not on the backup host and must never be put there, because beside the ciphertext it would make the encryption decorative. The file list is fetched from the VPS rather than duplicated here, so it tracks the host instead of drifting from a copy. - -**The log leg is the one with a deadline.** Those logs are deliberately excluded from the encrypted archives, because that set is many full copies with no dedupe and an append-only file would be multiplied across all of them for no recovery benefit. So the VPS's own retention window is the only copy until this runs. - -## Three behaviors that look like bugs and are not - -- **`--delete` never reaches the log leg**, whatever is passed. It exists to mirror the VPS's archive window, and applying it to an append-only record would delete the only remaining copy at exactly the moment it became the only one. The option array is copied before `--delete` is appended, rather than filtered afterwards, because a filter is a thing to get wrong later. -- **Today's live log is never fetched.** It is still being appended to, so a copy is a torn prefix that the next run fetches again. Rotation is what makes a file eligible. Read the live file over SSH when the analysis covers today. -- **Nothing prunes the destination.** Because the log leg passes no `--delete`, anything the VPS renames or re-compresses after it has been pulled keeps its old name on the backup host permanently, and a count that walks the tree by filename double-counts the overlap. **Read a date from a line's `StartUTC`, never from the filename holding it.** The VPS keeps an append-only `RECONCILE.md` inside the archive directory so a rename travels with the data it explains. - -## Install - -```sh -ops/install.sh --check # derive, validate, print, write nothing -ops/install.sh # the same, then install -``` - -**Nothing is typed twice.** The address, both destinations, and the account are already known to this checkout, so `install.sh` copies them rather than asking: `VPS_SSH_HOST`, `BACKUP_ARCHIVE_ROOT` and `LOG_ARCHIVE_ROOT` come straight from `~/.secrets/Blog...env`, the account is whoever runs the script, the group is read from the destination, and the mount is resolved with `findmnt`. - -**Two derivations are worth knowing, because the obvious answer is wrong for both.** The group comes from the destination rather than from `id -gn`, since `Group=` sets the process's primary group and the account's own group is usually not the one owning the backup tree. And `RequiresMountsFor=` needs the mount point rather than the destination path below it. - -**`--check` needs no root and writes nothing.** It prints both generated files, reports whether each would be created or already matches, and proves the VPS answers over SSH. Run it first. It reports "needs root to compare" rather than "already correct" when it cannot read an existing file, because an installer that claims agreement it could not verify is the failure this is written against. - -Re-running is safe and is how a changed value is applied. Both generated files are rewritten every run, so the comparison is a report and a guard rather than a skip: a file that already matches says so, and one that differs stops the run until `--force`. - -**The timer's hour sits behind both producers on the VPS rather than beside them**, because the VPS rotates its log and writes its archive at times of its own. Pulling before the day's archive exists fetches the previous one and reports success, which is the failure mode that looks like a working backup. `Persistent=true` covers a host that was powered off when the timer should have fired. - -**Run both as the account that will own the backup, never under `sudo`**, and expect one password prompt for the privileged steps the installer calls itself. The pull authenticates with that account's SSH key and writes into a tree that account owns. Under `sudo` it uses root's identity, which the VPS does not trust, and starts mixing root-owned files into a user-owned backup tree. That is why it installs to `bin/` rather than `sbin/`, why the drop-in sets `User=`, and why both scripts refuse to start as root rather than warning about it. - -## Checking it, without trusting anything written down - -```sh -journalctl -u vps-backup-pull.service -o short-iso | grep done -systemctl list-timers vps-backup-pull.timer --all -``` - -**A journal with one entry is not evidence of one copy.** The script can be run directly as well as by its timer, and a direct run writes no service record. Directory mtimes on the backup host are the copy times, where the file mtimes are the VPS's, so those are what to read when establishing when something arrived. - -## Variables - -Every path is a variable, so a host states its own layout rather than editing a file git owns. [`example.env`](../.secrets/example.env) lists them and [`ENVIRONMENT.md`](../ENVIRONMENT.md) describes them. The three `systemd` settings that cannot come from an environment file are in [`vps-backup-pull.service.d-local.conf.example`](./vps-backup-pull.service.d-local.conf.example). - -**They are the same names Blog's own `~/.secrets/` file uses, which is the point.** `VPS_*` is something on the VPS and `*_ROOT` is something on this host, and the pull writes the two roots that the log review reads. One name per directory means the writing side and the reading side cannot disagree, and it is why `install.sh` copies rather than translates. Every value is described in [`ENVIRONMENT.md`](../ENVIRONMENT.md). - -## This directory is the source - -Edit the copy here and install it. Nothing else is a source, and a copy found on a host is an installed artifact rather than a place to make a change. diff --git a/ops/install.sh b/ops/install.sh deleted file mode 100755 index f5fb64a..0000000 --- a/ops/install.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -# Install the backup pull on this host, deriving every host-specific value from the -# environment file this repository already keeps. -# -# The pull needs four things this repository deliberately does not carry: an address, two destination paths, and the account to run as. -# The first three are already in ~/.secrets/Blog...env under the same names the pull itself uses, so this copies them rather than translating them, and the account is whoever runs this. -# -# There is no name mapping here, because both sides spell every shared value the same way. -# Keep it that way: a translation table is a thing to get wrong every time one side changes. -# -# Usage: ops/install.sh [--check] [--force] -# --check validate and print what would be written, touch nothing, need no root -# --force overwrite an existing config file whose contents differ -# -# RUN IT AS THE ACCOUNT THAT WILL OWN THE BACKUP, not under sudo. That account's name and -# its SSH key are what the unit is built around, and running this under sudo would record -# root. Individual privileged steps call sudo themselves. -set -euo pipefail - -CHECK=0 -FORCE=0 -while [[ $# -gt 0 ]]; do - case "$1" in - --check) - CHECK=1 - shift - ;; - --force) - FORCE=1 - shift - ;; - -h | --help) - sed -n '2,22p' "$0" - exit 0 - ;; - *) - printf 'ERROR: unknown argument %s\n' "$1" >&2 - exit 1 - ;; - esac -done - -die() { - printf 'ERROR: %s\n' "$*" >&2 - exit 1 -} -note() { printf ' %s\n' "$*"; } - -REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -ENV_FILE=${ENV_FILE:-$HOME/.secrets/Blog.local.production.env} -# The same rule deploy/make-release.sh applies, because the two read the same files and a name that means different things depending on where you stood is worse here: this one installs. -# A relative name resolves against ~/.secrets, so it means the same from any directory, and traversal is refused rather than resolved, since a relative name is meant to reach ~/.secrets. -case "$ENV_FILE" in -/*) ;; -*..*) die "ENV_FILE must not traverse: $ENV_FILE" ;; -*) ENV_FILE="$HOME/.secrets/$ENV_FILE" ;; -esac - -[[ $EUID -ne 0 ]] || die "do not run this under sudo -- run it as the account that will own the backup; it calls sudo for the steps that need it" -[[ -f $ENV_FILE ]] || die "$ENV_FILE does not exist (ENV_FILE overrides which file is read)" - -# Sourced the same way every other script here reads it, so a value set by hand in the -# caller's shell does not quietly win over the file that is supposed to be authoritative. -set -a -# shellcheck disable=SC1090 -. "$ENV_FILE" -set +a - -[[ -n ${VPS_SSH_HOST:-} ]] || die "VPS_SSH_HOST is not set in $ENV_FILE" -[[ -n ${BACKUP_ARCHIVE_ROOT:-} ]] || die "BACKUP_ARCHIVE_ROOT is not set in $ENV_FILE" -[[ -n ${LOG_ARCHIVE_ROOT:-} ]] || die "LOG_ARCHIVE_ROOT is not set in $ENV_FILE" - -ACCOUNT=$(id -un) - -# The nearest existing ancestor of the destination, which is what both the group and the -# mount are read from. The destination itself may not exist on a first install. -ANCESTOR=$BACKUP_ARCHIVE_ROOT -while [[ ! -e $ANCESTOR && $ANCESTOR != / ]]; do ANCESTOR=$(dirname "$ANCESTOR"); done - -# The group comes from the destination rather than from `id -gn`, which is the wrong answer -# whenever the account's primary group is not the group that owns the backup tree. Group= -# sets the process's primary group, so getting it from the user would create files the -# existing tree's group cannot read, and only on the paths where a setgid bit does not -# already override it -- so it would half work, which is worse than failing. -GROUP=$(stat -c %G "$ANCESTOR") - -# The mount the destination sits on, resolved rather than guessed, because RequiresMountsFor -# has to name the mount point and not the directory below it. -MOUNT=$(findmnt -no TARGET --target "$ANCESTOR" 2>/dev/null) || - die "cannot resolve the mount holding $BACKUP_ARCHIVE_ROOT" - -VPS_TRAEFIK_ARCHIVE=${VPS_TRAEFIK_LOG_ARCHIVE:-/var/log/traefik/archive} - -# A display value only, built without the \~ parameter-expansion escape, since bash versions differ on whether that yields a literal backslash, and this is what an operator reads to trust the source file. -ENV_FILE_DISPLAY=$ENV_FILE -case "$ENV_FILE_DISPLAY" in -"$HOME"/*) - # shellcheck disable=SC2088 # Literal display text, deliberately not expanded. - ENV_FILE_DISPLAY="~/${ENV_FILE_DISPLAY#"$HOME"/}" - ;; -esac - -printf '=== derived from %s\n' "$ENV_FILE_DISPLAY" -note "VPS_SSH_HOST $VPS_SSH_HOST" -note "BACKUP_ARCHIVE_ROOT $BACKUP_ARCHIVE_ROOT" -note "LOG_ARCHIVE_ROOT $LOG_ARCHIVE_ROOT" -note "VPS_TRAEFIK_LOG_ARC. $VPS_TRAEFIK_ARCHIVE" -note "account $ACCOUNT:$GROUP" -note "mount $MOUNT" - -ENV_DEST=/etc/vps-backup-pull.env -DROPIN_DIR=/etc/systemd/system/vps-backup-pull.service.d -DROPIN_DEST=$DROPIN_DIR/local.conf - -ENV_BODY=$( - cat </dev/null; then - printf '=== %s: exists, contents need root to compare\n' "$path" - changed=1 - elif printf '%s\n' "$body" | sudo -n diff -q - "$path" >/dev/null 2>&1; then - printf '=== %s: already correct\n' "$path" - else - printf '=== %s: differs\n' "$path" - printf '%s\n' "$body" | sudo -n diff -u "$path" - || true - # --force is an install-path guard, not a reporting one. --check exists to show what - # would happen, so stopping at the first differing file would hide the second one and - # the reachability test behind it. - if [[ $CHECK -eq 0 && $FORCE -eq 0 ]]; then - die "$path exists with different contents -- re-run with --force to replace it" - fi - changed=1 - fi -done - -if [[ $CHECK -eq 1 ]]; then - printf '\n=== %s would contain\n' "$ENV_DEST" - printf '%s\n' "$ENV_BODY" | sed 's/^/ /' - printf '\n=== %s would contain\n' "$DROPIN_DEST" - printf '%s\n' "$DROPIN_BODY" | sed 's/^/ /' - printf '\n=== check only -- nothing written\n' - printf '=== verifying the VPS is reachable as %s\n' "$ACCOUNT" - # if/else rather than `A && B || C`, which runs C when B fails as well as when A does. - # shellcheck disable=SC2086 - if ssh ${SSH_OPTS:--o ConnectTimeout=15 -o BatchMode=yes} "$VPS_SSH_HOST" true; then - printf ' reachable\n' - else - die "cannot reach $VPS_SSH_HOST over SSH as $ACCOUNT (key auth only)" - fi - exit 0 -fi - -printf '=== installing\n' -sudo install -m 755 "$REPO/ops/vps-backup-pull" /usr/local/bin/vps-backup-pull -sudo install -m 644 "$REPO/ops/vps-backup-pull.service" "$REPO/ops/vps-backup-pull.timer" \ - /etc/systemd/system/ -printf '%s\n' "$ENV_BODY" | sudo install -m 600 /dev/stdin "$ENV_DEST" -sudo mkdir -p "$DROPIN_DIR" -printf '%s\n' "$DROPIN_BODY" | sudo install -m 644 /dev/stdin "$DROPIN_DEST" -sudo systemctl daemon-reload -sudo systemctl enable --now vps-backup-pull.timer - -printf '=== verifying the installed copy, without writing a journal entry\n' -# --dry-run rather than starting the service, because a service record is the evidence that -# the timer fired, and manufacturing one here would spend that evidence to prove the install. -/usr/local/bin/vps-backup-pull --dry-run >/dev/null || - die "the installed script failed its dry run -- $ENV_DEST or the drop-in is wrong" -printf ' dry run clean\n' -systemctl list-timers vps-backup-pull.timer --all --no-pager -if [[ $changed -eq 1 ]]; then - printf '=== done\n' -else - printf '=== done -- configuration was already correct\n' -fi diff --git a/ops/vps-backup-pull b/ops/vps-backup-pull deleted file mode 100755 index b5b01eb..0000000 --- a/ops/vps-backup-pull +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/env bash -# vps-backup-pull -- pull the Pangolin VPS backup set to this host. -# -# RUNS ON THE BACKUP HOST, NOT on the VPS. It installs as -# /usr/local/bin/vps-backup-pull, which is the path vps-backup-pull.service runs -# and the only path that matters. Installing to /usr/local/sbin/ instead leaves the -# timer running the previous copy and reports no error while doing it. -# -# sudo install -m 755 ops/vps-backup-pull /usr/local/bin/vps-backup-pull -# -# RUN IT AS YOUR NORMAL USER, NOT WITH SUDO. It authenticates to the VPS with that -# user's SSH key and writes to a tree that user owns. Under sudo it uses root's -# identity, which the VPS does not trust, and starts mixing root-owned files into a -# user-owned backup tree. That is why it lives in bin/ rather than sbin/, and why the -# shipped unit is a system timer that sets User= in a drop-in rather than running as root. -# A user timer or that account's crontab works equally well. What matters is the account, -# not which scheduler owns it. -# -# The source is ops/vps-backup-pull in the Blog repository. Edit it there and install -# it. A copy found on a host is an installed artifact, not a place to make a change. -# -# Paths below come from the environment, and this script reads nothing but the environment. -# Under the timer, systemd loads /etc/vps-backup-pull.env through the unit's EnvironmentFile. -# Run by hand, nothing loads it for you, so source an environment file first or the required -# values are unset. See example.env and ENVIRONMENT.md. -# -# Why a PULL and not a push from the VPS: it keeps the VPS free of any credential -# reaching another system. That rule is item 0 of the VPS's own /root/CONFIG.md, which is -# on that host rather than in this repository, and OPERATIONS.md "Working With the VPS" -# states the same reasoning here. The backup host holds a key the VPS trusts, and the VPS -# holds none for here. -# -# What it fetches: -# pangolin/ the encrypted archives -- the actual backup. Contains config/, -# docker-compose.yml, hostconfig/ and MANIFEST.txt. -# hostconfig/ a PLAINTEXT copy of the same non-secret host files, so the -# rebuild surface stays readable WITHOUT the encryption key. The -# file list is fetched from the VPS (pangolin-backup --list-files) -# rather than duplicated here, so it cannot drift. -# logs the rotated access logs, which are deliberately NOT in the -# encrypted archives -- see section 4 for why, and why this leg -# never mirrors deletions even under --delete. -# -# The encryption key is NOT fetched and must never live here -- beside the -# ciphertext it would make the encryption decorative. It is in 1Password. -# -# Every path is an environment variable, read from /etc/vps-backup-pull.env, so a host -# states its own layout there rather than editing a file git owns. The variables are listed -# in example.env and described in ENVIRONMENT.md. -# -# Usage: vps-backup-pull [--delete] [--no-hostconfig] [--no-logs] [--dry-run] [--quiet] -# --delete mirror deletions (Proxmox tracks the VPS's 14-archive window -# instead of growing without limit). Safe only because -# duplicacy keeps snapshot history in B2 -- without that, a -# deletion on the VPS would propagate irreversibly. It applies -# to the archives and host config ONLY, never to the logs. -# --no-hostconfig archives only -# --no-logs skip the access logs -# --dry-run show what would transfer, change nothing -# --quiet errors and the summary only -set -euo pipefail - -# No defaults, deliberately: preflight refuses to run without them rather than falling back -# to something plausible. They come from /etc/vps-backup-pull.env, which ops/install.sh -# generates from the same names in secrets/..env. -VPS_SSH_HOST=${VPS_SSH_HOST:-} -BACKUP_ARCHIVE_ROOT=${BACKUP_ARCHIVE_ROOT:-} -# Layout on the VPS, which is the same for any host running this stack, so these do default. -VPS_ARCHIVE_DIR=${VPS_ARCHIVE_DIR:-/var/backups/pangolin} -SSH_OPTS=${SSH_OPTS:--o ConnectTimeout=15 -o BatchMode=yes} - -# The log leg. Separate from BACKUP_ARCHIVE_ROOT because these are plaintext and long-lived: -# VPS prunes at 400 days and this host is the copy that outlives that window, so -# they must not share a directory whose retention tracks the VPS's 14 archives. -VPS_TRAEFIK_LOG_ARCHIVE=${VPS_TRAEFIK_LOG_ARCHIVE:-/var/log/traefik/archive} -VPS_BLOG_LOG_DIR=${VPS_BLOG_LOG_DIR:-/var/log/blog/legacy} -LOG_ARCHIVE_ROOT=${LOG_ARCHIVE_ROOT:-} - -DELETE=0 -HOSTCONFIG=1 -LOGS=1 -DRYRUN=0 -QUIET=0 -LOGFILES=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --delete) - DELETE=1 - shift - ;; - --no-hostconfig) - HOSTCONFIG=0 - shift - ;; - --no-logs) - LOGS=0 - shift - ;; - --dry-run) - DRYRUN=1 - shift - ;; - --quiet) - QUIET=1 - shift - ;; - -h | --help) - awk 'NR>1 && /^#/{sub(/^# ?/,""); print; next} NR>1{exit}' "$0" - exit 0 - ;; - *) - echo "unknown argument: $1" >&2 - exit 2 - ;; - esac -done - -say() { [[ $QUIET -eq 1 ]] || printf '%s\n' "$*"; } -warn() { printf 'WARNING: %s\n' "$*" >&2; } -die() { - printf 'ERROR: %s\n' "$*" >&2 - exit 1 -} - -# The closing summary, which prints even under --quiet. It went through say() before, so --quiet -# silenced it too and delivered neither half of the "errors and the summary only" this script's -# own usage promises. That matters most exactly where --quiet is used: under the timer, where a -# successful run then left a journal saying it started and finished and nothing about whether it -# copied anything, which is a backup with no evidence. -tell() { printf '%s\n' "$*"; } - -# Declared up front and cleaned up by ONE trap. Both are conditional -- --dry-run -# skips the checksum step, so a trap referencing an unset $SUMS would fail under -# `set -u` at exit and mask the real exit status. -SUMS="" -LIST="" -# shellcheck disable=SC2329 # invoked by the EXIT trap below, which shellcheck cannot see -cleanup() { - [[ -n $SUMS ]] && rm -f "$SUMS" - [[ -n $LIST ]] && rm -f "$LIST" - return 0 -} -trap cleanup EXIT - -RSYNC_OPTS=(-a --human-readable) -[[ $QUIET -eq 1 ]] || RSYNC_OPTS+=(--info=stats1) -# --itemize-changes rides with --dry-run because otherwise a dry run prints byte -# counts and no filenames, which proves the SSH and rsync legs work and previews -# nothing. "What would tomorrow's timed run bring?" is the only question a dry run -# is asked, and stats1 alone cannot answer it. -[[ $DRYRUN -eq 1 ]] && RSYNC_OPTS+=(--dry-run --itemize-changes) -# Taken before --delete is appended rather than filtered out afterwards, because a -# filter is a thing to get wrong later and this is a copy of two words. -LOG_RSYNC_OPTS=("${RSYNC_OPTS[@]}") -[[ $DELETE -eq 1 ]] && RSYNC_OPTS+=(--delete) - -# ---------------------------------------------------------------- preflight -# Ahead of the banner deliberately. A banner printed with empty values reads, in a journal -# skim, exactly like a run that started and then failed somewhere real, so nothing that -# looks like progress is printed until the configuration is known good. -# -# Each variable is named individually rather than reported as "configuration missing", -# which sends you to the wrong file when only one line of it is absent. LOG_ARCHIVE_ROOT is -# required only when the log leg is running, so --no-logs still works on a host that -# copies archives and nothing else. -# -# Root is refused rather than warned about. It authenticates with an identity the VPS does -# not trust, so the run fails regardless, but it fails after creating root-owned -# directories inside a tree the real account owns, and the next ordinary run then fails on -# those instead. The unit omits User= deliberately, so this is what catches a missing -# drop-in rather than the failure surfacing a day later as a permissions error. -[[ $EUID -ne 0 ]] || die "refusing to run as root -- run as the account that owns the destination and holds the VPS key (systemd: set User= in the drop-in)" -[[ -n $VPS_SSH_HOST ]] || die "VPS_SSH_HOST is not set -- see /etc/vps-backup-pull.env (template: example.env)" -[[ -n $BACKUP_ARCHIVE_ROOT ]] || die "BACKUP_ARCHIVE_ROOT is not set -- see /etc/vps-backup-pull.env (template: example.env)" -[[ $LOGS -eq 0 || -n $LOG_ARCHIVE_ROOT ]] || die "LOG_ARCHIVE_ROOT is not set and the log leg is enabled -- set it, or pass --no-logs" - -# Both roots are chmod 700'd and written into, so a wrong value here is not a failed backup, -# it is damage to the host. `/` is the case that matters: `chmod 700 /` locks every other -# account out of the filesystem, and nothing downstream would refuse it. A relative value is -# refused for the same reason it is refused in the deploy tooling, since it means a different -# directory depending on where the caller stood, and under the timer that is systemd's cwd. -check_root() { - local name=$1 value=$2 - [[ $value == /* ]] || die "$name must be an absolute path, got: $value" - [[ $value != "/" ]] || die "$name must not be / -- this directory is chmod 700'd and written into" - [[ $value != */ ]] || die "$name must not end in a slash, got: $value" -} -check_root BACKUP_ARCHIVE_ROOT "$BACKUP_ARCHIVE_ROOT" -[[ $LOGS -eq 0 ]] || check_root LOG_ARCHIVE_ROOT "$LOG_ARCHIVE_ROOT" - -command -v rsync >/dev/null || die "rsync not installed on this host" - -START=$(date -u '+%Y-%m-%d %H:%M:%S UTC') -say "=== VPS backup pull -- $START" -say " source: $VPS_SSH_HOST dest: $BACKUP_ARCHIVE_ROOT" -[[ $LOGS -eq 1 ]] && say " logs: $LOG_ARCHIVE_ROOT" -[[ $DRYRUN -eq 1 ]] && say " DRY RUN -- nothing will be written" -# shellcheck disable=SC2086 -ssh $SSH_OPTS "$VPS_SSH_HOST" true 2>/dev/null || - die "cannot reach $VPS_SSH_HOST over SSH (key auth only -- password auth is disabled there)" - -if [[ $DRYRUN -eq 0 ]]; then - mkdir -p "$BACKUP_ARCHIVE_ROOT/pangolin" || die "cannot create $BACKUP_ARCHIVE_ROOT/pangolin" - chmod 700 "$BACKUP_ARCHIVE_ROOT" -fi - -# ---------------------------------------------------------------- 1. archives -say -say "--- archives" -# shellcheck disable=SC2086 -rsync "${RSYNC_OPTS[@]}" -e "ssh $SSH_OPTS" \ - "$VPS_SSH_HOST:$VPS_ARCHIVE_DIR/" "$BACKUP_ARCHIVE_ROOT/pangolin/" || - die "archive rsync failed" - -# ---------------------------------------------------------------- 2. verify -# rsync verifies its own transfers, but this re-reads what actually landed on -# disk. A backup that was never independently checked is a hope, not a backup. -VERIFY_RC=0 -# Reported in the closing summary, so a scheduled run records how many archives were -# re-read from disk rather than only that it finished. Zero on a dry run, which prints -# its own summary and never reaches that line. -VERIFIED=0 -if [[ $DRYRUN -eq 1 ]]; then - say " (dry run -- checksum verification skipped)" -else - say - say "--- verifying checksums" - SUMS=$(mktemp) - # SC2086: SSH_OPTS is a word list and must split. - # SC2029: VPS_ARCHIVE_DIR expanding here is the point, since the configured path is - # this side's. The *.enc glob stays quoted so the remote shell expands it instead. - # shellcheck disable=SC2086,SC2029 - ssh $SSH_OPTS "$VPS_SSH_HOST" "cd $VPS_ARCHIVE_DIR && sha256sum *.enc" >"$SUMS" || - die "could not read source checksums" - EXPECTED=$(wc -l <"$SUMS") - if OUTPUT=$(cd "$BACKUP_ARCHIVE_ROOT/pangolin" && sha256sum -c "$SUMS" 2>&1); then - VERIFIED=$EXPECTED - say " $EXPECTED/$EXPECTED archives verified OK" - else - VERIFY_RC=1 - printf '%s\n' "$OUTPUT" | grep -v ': OK$' >&2 || true - warn "checksum verification FAILED -- the local copy does not match the VPS" - fi -fi - -# ---------------------------------------------------------------- 3. hostconfig -if [[ $HOSTCONFIG -eq 1 ]]; then - say - say "--- host config (plaintext, readable without the key)" - LIST=$(mktemp) - # Ask the VPS what it considers valuable. Single source of truth: the list is - # defined once, in pangolin-backup's HOST_FILES, and never copied to this host. - # shellcheck disable=SC2086 - if ssh $SSH_OPTS "$VPS_SSH_HOST" 'pangolin-backup --list-files' 2>/dev/null | - sed 's|^/||' >"$LIST" && [[ -s $LIST ]]; then - WANT=$(wc -l <"$LIST") - say " $WANT path(s) advertised by the VPS" - [[ $DRYRUN -eq 0 ]] && mkdir -p "$BACKUP_ARCHIVE_ROOT/hostconfig" - # shellcheck disable=SC2086 - rsync "${RSYNC_OPTS[@]}" -e "ssh $SSH_OPTS" \ - --files-from="$LIST" "$VPS_SSH_HOST:/" "$BACKUP_ARCHIVE_ROOT/hostconfig/" || - warn "host config rsync reported errors" - if [[ $DRYRUN -eq 0 ]]; then - GOT=$(find "$BACKUP_ARCHIVE_ROOT/hostconfig" -type f | wc -l) - say " $GOT file(s) on disk" - [[ $GOT -eq $WANT ]] || warn "expected $WANT file(s), found $GOT -- a listed path may be missing on the VPS" - fi - else - warn "could not get the file list from the VPS (old pangolin-backup without --list-files?) -- skipping host config" - fi -fi - -# ---------------------------------------------------------------- 4. access logs -# Deliberately NOT in the encrypted archives: pangolin-backup keeps fourteen full -# copies, each encrypted with a fresh salt so Backblaze cannot dedupe them, and an -# append-only file that grows forever would be multiplied by fourteen for no -# recovery benefit. So this is the ONLY copy that outlives the VPS's 400 days. -# -# NEVER --delete here, whatever was passed. The VPS prunes at 400 days by design, -# and this host is the long-term copy; mirroring that prune would delete the only -# remaining copy at exactly the moment it became the only one. Both sources are -# immutable once written, so the transfer is genuinely incremental either way. -# -# Today's live access.log is NOT fetched. It is still being appended to, so a copy -# is a torn prefix that the next run would fetch again; the rotation at 00:00 UTC -# is what makes a file eligible. Read the live file over SSH when analysing today. -if [[ $LOGS -eq 1 ]]; then - say - say "--- access logs (plaintext, and the only copy past 400 days)" - if [[ $DRYRUN -eq 0 ]]; then - mkdir -p "$LOG_ARCHIVE_ROOT/traefik" "$LOG_ARCHIVE_ROOT/blog-legacy" || die "cannot create $LOG_ARCHIVE_ROOT" - # Query strings are logged in full, so the tree is as sensitive as the log is. - chmod 700 "$LOG_ARCHIVE_ROOT" - fi - # shellcheck disable=SC2086 - rsync "${LOG_RSYNC_OPTS[@]}" -e "ssh $SSH_OPTS" \ - "$VPS_SSH_HOST:$VPS_TRAEFIK_LOG_ARCHIVE/" "$LOG_ARCHIVE_ROOT/traefik/" || - warn "traefik log rsync reported errors" - # shellcheck disable=SC2086 - rsync "${LOG_RSYNC_OPTS[@]}" -e "ssh $SSH_OPTS" \ - "$VPS_SSH_HOST:$VPS_BLOG_LOG_DIR/" "$LOG_ARCHIVE_ROOT/blog-legacy/" || - warn "blog legacy log rsync reported errors" - if [[ $DRYRUN -eq 0 ]]; then - LOGFILES=$(find "$LOG_ARCHIVE_ROOT" -type f | wc -l) - say " $LOGFILES log file(s) on disk" - # A pull that lands nothing looks identical to a pull with nothing new, and the - # difference is a broken path against a working one. Only the empty case is odd. - [[ $LOGFILES -gt 0 ]] || warn "no log files landed -- check VPS_TRAEFIK_LOG_ARCHIVE and VPS_BLOG_LOG_DIR" - fi -fi - -# ---------------------------------------------------------------- summary -say -if [[ $DRYRUN -eq 1 ]]; then - # tell() rather than say(), because this IS the summary for a dry run, and --quiet - # promises errors and the summary. say() would make `--quiet --dry-run` print nothing. - tell "=== dry run complete -- nothing changed" - exit 0 -fi -ARCHIVES=$(find "$BACKUP_ARCHIVE_ROOT/pangolin" -name '*.enc' -type f | wc -l) -# `set -e` plus `pipefail` makes a failing du fatal, and 2>/dev/null then hides why, so a -# pull that copied everything exits 1 at the summary with an empty journal. The size is -# cosmetic, so let du report to stderr and carry on with the transfer already done. -SIZE=$(du -sh "$BACKUP_ARCHIVE_ROOT" | cut -f1) || SIZE="size unavailable" -if [[ $LOGS -eq 1 ]]; then - LOGSIZE=$(du -sh "$LOG_ARCHIVE_ROOT" | cut -f1) || LOGSIZE="size unavailable" - tell "=== done -- $ARCHIVES archive(s), $VERIFIED verified, $SIZE in $BACKUP_ARCHIVE_ROOT; $LOGFILES log file(s), $LOGSIZE in $LOG_ARCHIVE_ROOT" -else - tell "=== done -- $ARCHIVES archive(s), $VERIFIED verified, $SIZE in $BACKUP_ARCHIVE_ROOT; logs skipped" -fi -if [[ $VERIFY_RC -ne 0 ]]; then - tell "=== COMPLETED WITH ERRORS -- see warnings above" - exit 1 -fi -say " Restore needs the key from 1Password; it is deliberately not stored here." -exit 0 diff --git a/ops/vps-backup-pull.service b/ops/vps-backup-pull.service deleted file mode 100644 index 015059f..0000000 --- a/ops/vps-backup-pull.service +++ /dev/null @@ -1,43 +0,0 @@ -[Unit] -# Pulls the VPS's encrypted archives, its plaintext host config, and the rotated -# access logs. The logs are the reason this has a deadline rather than being a -# convenience: they are deliberately excluded from the encrypted archives, so the -# VPS's own retention window is the only copy until this lands. -Description=Pull the Pangolin VPS backup set and access logs to this host -Documentation=file:/usr/local/bin/vps-backup-pull -Documentation=https://github.com/ptr727/Blog/blob/main/ops/README.md -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -# The paths, the address, and the account are all host-specific, and this file is public, -# so none of them is written here. Two mechanisms supply them and both are required: -# -# /etc/vps-backup-pull.env the paths and the address -# /etc/systemd/system/vps-backup-pull.service.d/*.conf User=, Group=, RequiresMountsFor= -# -# Templates for both are in ops/, and ops/README.md has the install steps. The leading - -# below keeps systemd from failing the unit outright when the env file is absent, because -# the script's own preflight names the missing variable, which is the more useful error. -EnvironmentFile=-/etc/vps-backup-pull.env -# NOT set here, deliberately. A username belongs to one machine, and a wrong guess baked -# into a public file is worse than an absent one: the script refuses to run as root rather -# than authenticating with an identity the VPS does not trust and writing root-owned files -# into a user-owned tree. Set User= and Group= in the drop-in. -# -# RequiresMountsFor= belongs in the drop-in for the same reason, and it is not optional on -# a host whose destination is a mount: a run starting before the mount writes a full copy -# into the mountpoint underneath it, where nothing ever reads it and the space does not -# show up in du against the mounted path. -# -# --quiet prints errors and the one-line summary only, which is what belongs in a journal. -# Add --delete to mirror the VPS's archive window instead of growing without limit. It is -# deliberately not set: it is safe only where snapshot history exists off this host, and it -# never touches the logs whatever is passed. -ExecStart=/usr/local/bin/vps-backup-pull --quiet -# The first run transfers the whole archive set. -TimeoutStartSec=2h -# A backup pull is never the urgent thing on this host. -Nice=10 -IOSchedulingClass=idle diff --git a/ops/vps-backup-pull.service.d-local.conf.example b/ops/vps-backup-pull.service.d-local.conf.example deleted file mode 100644 index cb5842c..0000000 --- a/ops/vps-backup-pull.service.d-local.conf.example +++ /dev/null @@ -1,19 +0,0 @@ -# Copy to /etc/systemd/system/vps-backup-pull.service.d/local.conf on the backup host. -# The three settings here name one machine, which is why the unit in git does not carry them. -# -# systemd merges drop-ins over the unit, so a later install of the unit leaves this intact. - -[Unit] -# Required when the destination is on a mounted filesystem, which it usually is. -# A run that starts before the mount writes a full copy into the mountpoint underneath it, -# where nothing ever reads it and the space does not show up in du against the mounted path. -# Name the mount point, not the destination directory below it. -RequiresMountsFor=/path/to/mount - -[Service] -# The account whose SSH key the VPS trusts and that owns the destination tree. -# Not optional: without it the unit runs as root, and the script refuses to start, because -# root authenticates with an identity the VPS does not trust and writes root-owned files -# into a user-owned tree. -User=someuser -Group=somegroup diff --git a/ops/vps-backup-pull.timer b/ops/vps-backup-pull.timer deleted file mode 100644 index 43e70ae..0000000 --- a/ops/vps-backup-pull.timer +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Daily VPS backup and access-log pull -Documentation=file:/usr/local/bin/vps-backup-pull - -[Timer] -# 09:00 UTC, chosen to sit behind both producers on the VPS rather than beside them: -# logrotate rotates the access log at 00:00 UTC, and pangolin-backup.timer writes the -# day's encrypted archive at 08:03 UTC. Running before the archive exists would pull -# yesterday's and report success, which is the failure that looks like a working backup. -# Stated in UTC explicitly, because this host runs local time and the VPS does not. -OnCalendar=*-*-* 09:00:00 UTC -# The VPS is a single small host; nothing here needs to hit it on the second. -RandomizedDelaySec=15m -# Runs on next boot if the host was down at 09:00. Without this a machine that is off -# overnight silently never backs up, and the gap is only visible by reading timestamps. -Persistent=true - -[Install] -WantedBy=timers.target