Stop an AUR outage from masking release verification

.github/workflows/aur-publish.yml

@@ -0,0 +1,141 @@
+# Manual AUR publish, for when the release-time publish could not run.
+#
+# The AUR goes down for maintenance often enough that a release must not depend
+# on it being reachable — release.yml's aur-publish job is continue-on-error for
+# exactly that reason. This workflow is the recovery path: it republishes any
+# already-released version once the AUR is back, with no need to cut a new tag.
+#
+# scripts/publish-aur.sh derives the PKGBUILD entirely from the published GitHub
+# release assets and no-ops when the AUR copy is already current, so running this
+# against an already-published version is safe and idempotent.
+name: Publish to AUR
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Released version to publish, without the v prefix (e.g. 0.8.0)'
+        required: true
+        type: string
+
+permissions: {}
+
+concurrency:
+  group: aur-publish
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    name: Publish to AUR
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    environment: release
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Validate version input
+        env:
+          VERSION: ${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::Invalid version '${VERSION}' — expected semver with no v prefix (e.g. 0.8.0)"
+            exit 1
+          fi
+          # Only publish versions that actually exist as a stable release, so a
+          # typo cannot push a PKGBUILD pointing at assets that were never built.
+          if ! gh release view "v${VERSION}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
+            echo "::error::No published release found for v${VERSION}"
+            exit 1
+          fi
+          echo "Publishing v${VERSION} to the AUR"
+
+      - name: Refuse to downgrade the AUR package
+        env:
+          VERSION: ${{ inputs.version }}
+        run: |
+          # publish-aur.sh rewrites pkgver unconditionally, so dispatching an
+          # older-but-valid release would push every Arch user backwards. Being
+          # a published release is not enough — compare against what is actually
+          # in the AUR right now.
+          if ! body=$(curl -fsS --max-time 30 --retry 2 \
+                        'https://aur.archlinux.org/rpc/v5/info/basecamp-cli'); then
+            # Fail closed: proceeding blind here risks a silent downgrade, and a
+            # dispatch is cheap to repeat once the AUR is reachable again.
+            echo "::error::Could not reach the AUR to check the published version. Retry when it is reachable."
+            exit 1
+          fi
+
+          # AUR reports version as pkgver-pkgrel (e.g. 0.7.2-1).
+          full=$(printf '%s' "$body" | jq -r '.results[0].Version // empty')
+          if [ -z "$full" ]; then
+            echo "basecamp-cli is not in the AUR yet — nothing to downgrade"
+            exit 0
+          fi
+          if [[ "$full" == *-* ]]; then
+            current="${full%-*}"
+            currel="${full##*-}"
+          else
+            current="$full"
+            currel=1
+          fi
+
+          if [ "$current" = "$VERSION" ]; then
+            # publish-aur.sh hardcodes pkgrel=1, so republishing over a higher
+            # revision would silently discard an AUR-side packaging fix.
+            #
+            # Compare with sort -V, not an arithmetic test: PKGBUILD(5) allows a
+            # dotted subrelease (1.1), which `[ -gt ]` rejects as a non-integer
+            # with status 2 — and a suppressed stderr would turn that error into
+            # a silent "not greater", letting the clobber through.
+            highest=$(printf '%s\n%s\n' "$currel" "1" | sort -V | tail -1)
+            if [ "$highest" != "1" ]; then
+              echo "::error::AUR has ${full}; publish-aur.sh would replace it with ${VERSION}-1 and discard that packaging revision"
+              exit 1
+            fi
+            echo "AUR already at ${full} — republish is a no-op unless the PKGBUILD drifted"
+            exit 0
+          fi
+
+          oldest=$(printf '%s\n%s\n' "$current" "$VERSION" | sort -V | head -1)
+          if [ "$oldest" = "$VERSION" ]; then
+            echo "::error::Refusing to downgrade the AUR from ${current} to ${VERSION}"
+            exit 1
+          fi
+          echo "AUR is at ${current}; publishing ${VERSION}"
+
+      - name: Verify AUR key is configured
+        env:
+          AUR_KEY: ${{ secrets.AUR_KEY }}
+        run: |
+          if [ -z "$AUR_KEY" ]; then
+            echo "::error::AUR_KEY is not configured for the release environment"
+            exit 1
+          fi
+
+      - name: Publish to AUR
+        env:
+          AUR_KEY: ${{ secrets.AUR_KEY }}
+          VERSION: ${{ inputs.version }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$AUR_KEY" > ~/.ssh/aur
+          chmod 600 ~/.ssh/aur
+          echo -e "Host aur.archlinux.org\n  IdentityFile ~/.ssh/aur\n  User aur\n  StrictHostKeyChecking accept-new" >> ~/.ssh/config
+          git config --global user.name "37signals"
+          git config --global user.email "dev@37signals.com"
+          for attempt in 1 2 3; do
+            if scripts/publish-aur.sh "$VERSION"; then
+              exit 0
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              echo "AUR publish attempt ${attempt} failed; retrying in $((attempt * 60))s"
+              sleep $((attempt * 60))
+            fi
+          done
+          echo "::error::AUR publish failed after 3 attempts"
+          exit 1

.github/workflows/release.yml

@@ -281,9 +281,36 @@ jobs:
         with:
           subject-checksums: ./dist/checksums.txt
 
+  # AUR publishing runs as its own job, not a step in `release`. As a step it
+  # sat *after* GoReleaser and attestation, so an AUR-side outage turned the
+  # whole job red with the release already published — and that took the four
+  # `needs: [release]` verification jobs down with it as skipped. Isolated and
+  # continue-on-error, an AUR outage can no longer mask signing verification.
+  # Recover a missed publish with the aur-publish workflow (workflow_dispatch).
+  aur-publish:
+    name: Publish to AUR
+    needs: [release]
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    # Shared with the aur-publish workflow's recovery runs. Without this, a
+    # recovery that already passed its version check could clone after a newer
+    # automatic publish landed and push the older PKGBUILD as a fast-forward.
+    concurrency:
+      group: aur-publish
+      cancel-in-progress: false
+    environment: release
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
       - name: Check AUR publishing configuration
-        id: aur-publish
-        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
+        id: aur-config
         env:
           AUR_KEY: ${{ secrets.AUR_KEY }}
         run: |
@@ -294,7 +321,7 @@ jobs:
           fi
 
       - name: Publish to AUR
-        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') && steps.aur-publish.outputs.enabled == 'true'
+        if: steps.aur-config.outputs.enabled == 'true'
         env:
           AUR_KEY: ${{ secrets.AUR_KEY }}
         run: |
@@ -305,7 +332,40 @@ jobs:
           echo -e "Host aur.archlinux.org\n  IdentityFile ~/.ssh/aur\n  User aur\n  StrictHostKeyChecking accept-new" >> ~/.ssh/config
           git config --global user.name "37signals"
           git config --global user.email "dev@37signals.com"
-          scripts/publish-aur.sh "$VERSION"
+          # publish-aur.sh derives everything from the published release assets
+          # and no-ops when the PKGBUILD is already current, so retrying is safe.
+          for attempt in 1 2 3; do
+            if scripts/publish-aur.sh "$VERSION"; then
+              exit 0
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              echo "AUR publish attempt ${attempt} failed; retrying in $((attempt * 60))s"
+              sleep $((attempt * 60))
+            fi
+          done
+          echo "AUR publish failed after 3 attempts"
+          exit 1
+
+      - name: Notify on AUR publish failure
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          TITLE="AUR publish failure"
+          BODY="Publishing [${REF_NAME}](https://github.com/${{ github.repository }}/actions/runs/${RUN_ID}) to the AUR failed. The GitHub release is unaffected — only the Arch package is stale. Re-run the \`Publish to AUR\` workflow with version \`${REF_NAME#v}\` once the AUR is reachable."
+
+          # Check for existing open issue before creating a new one
+          existing=$(gh issue list --repo ${{ github.repository }} --state open --search "in:title $TITLE" --json number,title --jq '[.[] | select(.title == "'"$TITLE"'")][0].number // empty' 2>/dev/null || true)
+          if [ -n "$existing" ]; then
+            gh issue comment --repo ${{ github.repository }} "$existing" --body "$BODY" || true
+          else
+            gh issue create --repo ${{ github.repository }} --title "$TITLE" --body "$BODY" || true
+          fi
+
+          # Always emit annotation so the failure is visible in the workflow summary
+          echo "::error::AUR publish failed for ${REF_NAME}. See https://github.com/${{ github.repository }}/actions/runs/${RUN_ID}"
 
   macos-verify:
     name: Verify macOS signing

.github/workflows/sync-skills.yml

@@ -0,0 +1,104 @@
+# Manual skills sync, for when the release-time sync could not run.
+#
+# The sync-skills job in release.yml is `needs: [release]`, so anything that
+# fails the release job after publication — an unreachable AUR, say — skips the
+# sync entirely and leaves basecamp/skills stale against a shipped release.
+# release.yml no longer fails that way, but a skipped or failed sync still needs
+# a way back without cutting a new tag. This is it.
+#
+# scripts/sync-skills.sh mirrors the skills/ tree at the given ref into
+# basecamp/skills and no-ops when the content already matches, so re-running it
+# for an already-synced release is safe.
+name: Sync skills
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag to sync from, with the v prefix (e.g. v0.8.0)'
+        required: true
+        type: string
+      dry_run:
+        description: 'Preview the sync without pushing'
+        required: false
+        default: false
+        type: boolean
+
+permissions: {}
+
+concurrency:
+  group: sync-skills
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    name: Sync skills
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: release
+    permissions:
+      contents: read
+    steps:
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # No prerelease suffix: the automatic sync in release.yml excludes
+          # prereleases deliberately, and this path must not smuggle prerelease
+          # content into the distribution repo's main branch.
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::Invalid tag '${TAG}' — expected a stable v-prefixed semver tag (e.g. v0.8.0)"
+            exit 1
+          fi
+
+          # sync-skills.sh mirrors the tree wholesale, so syncing an older tag
+          # would roll basecamp/skills back. Unlike the AUR, there is no
+          # independent record of what the distribution repo currently holds, and
+          # the only reason to run this by hand is that the newest release failed
+          # to sync — so require exactly that release.
+          latest=$(gh release view --repo "${GITHUB_REPOSITORY}" --json tagName --jq .tagName)
+          if [ "$TAG" != "$latest" ]; then
+            echo "::error::Refusing to sync ${TAG}: the latest stable release is ${latest}. Syncing an older tag would roll basecamp/skills back."
+            exit 1
+          fi
+
+      # Check out the tag itself, not main: the sync must mirror the skills tree
+      # as it was released, even if main has moved on since.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.tag }}
+          persist-credentials: false
+
+      # Needed for dry runs too: the honest preview clones the target, so it
+      # cannot run tokenless. Dry runs get a read-only token, which is also what
+      # stops DRY_RUN=remote from pushing even if the script were wrong.
+      - name: Generate token for skills repo
+        id: skills-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.RELEASE_CLIENT_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+          owner: basecamp
+          repositories: skills
+          permission-contents: ${{ inputs.dry_run && 'read' || 'write' }}
+
+      # github.sha is the dispatching ref's SHA (main), not the tag's, so resolve
+      # the commit actually checked out — otherwise the sync records the wrong
+      # provenance for the release it claims to mirror.
+      - name: Resolve the tagged commit
+        id: source
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      # DRY_RUN=remote, not local: the local path never clones basecamp/skills
+      # and diffs against an empty repo, so every skill reads as newly added and
+      # the deletions a real sync would make never appear. That is the opposite
+      # of what a preview is for. Remote clones the actual target and stops
+      # before the push.
+      - name: Sync skills to distribution repo
+        env:
+          SKILLS_TOKEN: ${{ steps.skills-token.outputs.token }}
+          RELEASE_TAG: ${{ inputs.tag }}
+          SOURCE_SHA: ${{ steps.source.outputs.sha }}
+          DRY_RUN: ${{ inputs.dry_run && 'remote' || '' }}
+        run: scripts/sync-skills.sh