fix(release): retry the cosign call, bounded and transient-only (backend#2379)

.github/workflows/build.yml

@@ -79,6 +79,10 @@ jobs:
  shellcheck --shell=bash --severity=error scripts/check-tool-pins.sh
  shellcheck --shell=bash --severity=error scripts/format.sh
  shellcheck --shell=bash --severity=error scripts/tests/format-verify.sh
+ shellcheck --shell=bash --severity=error scripts/cosign-retry.sh
+ shellcheck --shell=bash --severity=error scripts/tests/cosign-retry-verify.sh
+ bash -n scripts/cosign-retry.sh
+ bash -n scripts/tests/cosign-retry-verify.sh
  dash -n scripts/install.sh
  bash -n scripts/tests/install-verify.sh
  shellcheck --shell=bash --severity=error scripts/tests/install-ps1-verify.sh
@@ -92,6 +96,13 @@ jobs:
 
  - name: Verification harness (mandatory cosign / fail-closed)
  run: bash scripts/tests/install-verify.sh
+ # release.yml's cosign call is bounded, retries only transient sigstore
+ # failures, and fails closed on a zero exit with no signature
+ # (backend#2379). cosign is stubbed by a PATH shim, so this is hermetic
+ # and needs no network — which is why it sits in this job rather than
+ # Release, where it could only be exercised by cutting a tag.
+ - name: Cosign retry harness (bounded / classified / fail-closed)
+ run: bash scripts/tests/cosign-retry-verify.sh
  # Same property on Windows (backend#2078). pwsh is preinstalled on the
  # ubuntu runner image; the harness FAILS rather than skips if it isn't,
  # since "cannot tell" is not evidence that verification is mandatory.

.github/workflows/release.yml

@@ -165,6 +165,19 @@ jobs:
  ls -lh "dist/$BIN_NAME"
 
  - name: Sign binary with cosign (keyless)
+ # Via scripts/cosign-retry.sh, NOT a bare `cosign` (backend#2379).
+ # sign-blob is a call to a third-party CDN with no local alternative,
+ # and on 2026-08-23 a single sigstore TUF reset on this one leg left
+ # v0.10.10-rc.2 a git tag with no Release and no assets for 6h04m —
+ # because `publish` below is (correctly) gated on ALL eight legs.
+ #
+ # The retry is deliberately HERE, on the one network call, and not a
+ # job-level re-run: re-running redoes seven good legs and widens the
+ # very window this closes. The `needs: release` gate on `publish` is
+ # NOT the bug and must stay — a release missing darwin/arm64 is worse
+ # than no release — so the wrapper fails closed: bounded attempts,
+ # transient-only retries, and a zero exit with no .sig still fails.
+ # Properties pinned by scripts/tests/cosign-retry-verify.sh.
  env:
  # Required for keyless signing in GHA. The id-token: write
  # permission above grants the workflow an OIDC token Sigstore
@@ -173,7 +186,7 @@ jobs:
  run: |
  BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}"
  cd dist
- cosign sign-blob \
+"$GITHUB_WORKSPACE/scripts/cosign-retry.sh" sign-blob \
  --output-certificate "$BIN_NAME.cert" \
  --output-signature "$BIN_NAME.sig" \
  "$BIN_NAME"

scripts/RELEASE_CHECKLIST.md

@@ -64,6 +64,29 @@ The workflow takes 5-10 minutes. Monitor at
 - [ ] install.sh + install.ps1 present
 - [ ] Each binary has a `.cert` + `.sig` pair
 
+**If there is no GitHub Release at all** — the tag exists and nothing is
+attached — one build leg failed, and `publish` is gated on all eight, by
+design (a release missing `darwin/arm64` is worse than no release). That is
+the backend#2379 shape: on 2026-08-23 a sigstore TUF CDN reset failed the
+darwin/arm64 leg and `v0.10.10-rc.2` sat as a tag with zero assets for
+6h04m. Recover with:
+
+```bash
+gh run rerun <run-id> --repo tracebloc/cli --failed
+```
+
+Re-run the failed jobs, or dispatch the workflow **at the tag ref**. Never
+dispatch from a branch: cosign embeds the run's ref in the keyless identity
+and the installers trust only `@refs/tags/v.*`, so a branch-dispatched
+"rebuild of a tag" publishes signatures every customer install rejects
+(release.yml's own comment; Bugbot on promotion #428).
+
+Transient sigstore failures are now retried in-place by
+`scripts/cosign-retry.sh` (bounded, transient-only), so this recovery should
+be rare. If a leg still fails, read its log before re-running: the wrapper
+does **not** retry a genuine signing refusal, and re-running one of those
+just fails again.
+
 ### 4. Sanity-test each install path on a clean host
 
 ```bash

scripts/cosign-retry.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+# =============================================================================
+# cosign-retry.sh — run ONE cosign invocation with a bounded, classified retry.
+#
+# Why this exists (backend#2379). On 2026-08-23 tag v0.10.10-rc.2 was cut, and
+# one of release.yml's eight matrix legs — darwin/arm64 — died on:
+#
+# signing tracebloc-v0.10.10-rc.2-darwin-arm64: getting key from Fulcio:
+# getting CTFE public keys: updating local metadata and targets:
+# error updating to TUF remote mirror: tuf: failed to download 10.root.json:
+# Get "https://tuf-repo-cdn.sigstore.dev/10.root.json": read tcp ...:
+# read: connection reset by peer
+#
+# A sigstore CDN reset. The tag is the workflow's TRIGGER, so it already
+# existed; the release object is created by the `publish` job, gated on all
+# eight legs. One flaky leg therefore meant a git tag with no GitHub Release
+# and no assets, for 6h04m, until a manual re-run. fr-assist caught it, not
+# the build.
+#
+# The gate is NOT the bug and must not be weakened: a release missing
+# darwin/arm64 is worse than no release. The unretried network call is the bug,
+# and it is retried HERE — one cosign call — not by re-running the job, which
+# would redo seven good legs and widen the window.
+#
+# THREE PROPERTIES, each pinned by scripts/tests/cosign-retry-verify.sh:
+#
+# 1. BOUNDED. At most COSIGN_RETRY_ATTEMPTS attempts (default 3), with a
+# declared backoff (default 5s then 15s) and a per-attempt wall-clock cap
+# (default 120s, enforced in bash so it is verifiable on every host) so a
+# cosign that hangs against a degraded CDN is killed and retried rather
+# than eating the job budget. Worst case with the shipped defaults is
+# 3x120s + 5s + 15s = 6m20s, inside release.yml's timeout-minutes: 20 —
+# so the retry can never be the reason a leg is killed.
+#
+# 2. CLASSIFIED. Only failures whose output matches a known transient
+# sigstore/network signature are retried. A genuine signing refusal — a
+# missing file, a rejected OIDC token, a bad flag — fails on the FIRST
+# attempt. An indiscriminate retry would turn a real signing failure into a
+# slow red (or worse, hide it behind noise); "retry everything" is the
+# failure mode this script is written to avoid.
+#
+# 3. FAIL CLOSED. cosign exiting 0 is not evidence that it signed anything.
+# Every --output-* file named on the command line must exist and be
+# non-empty afterwards, or this script fails — and does NOT retry, because
+# a zero exit with no artifact is not a network symptom. "Cannot tell" is a
+# finding, never a pass.
+#
+# Usage (arguments are passed to cosign verbatim):
+# scripts/cosign-retry.sh sign-blob \
+# --output-certificate "$BIN.cert" --output-signature "$BIN.sig" "$BIN"
+#
+# Exit codes: 0 signed · 1 cosign failed (classified, or budget exhausted)
+# 2 usage/config error · 3 cosign exited 0 but produced no artifact
+#
+# Env (all optional): COSIGN_BIN COSIGN_RETRY_ATTEMPTS COSIGN_RETRY_DELAYS
+# COSIGN_RETRY_TIMEOUT
+# =============================================================================
+set -uo pipefail
+
+COSIGN_BIN="${COSIGN_BIN:-cosign}"
+ATTEMPTS="${COSIGN_RETRY_ATTEMPTS:-3}"
+DELAYS="${COSIGN_RETRY_DELAYS:-5 15}"
+PER_ATTEMPT_TIMEOUT="${COSIGN_RETRY_TIMEOUT:-120}"
+
+die() { printf 'cosign-retry: %s\n' "$1" >&2; exit "${2:-2}"; }
+
+[ "$#" -gt 0 ] || die "no cosign arguments given"
+
+case "$ATTEMPTS" in
+ ''|*[!0-9]*) die "COSIGN_RETRY_ATTEMPTS must be a positive integer, got '$ATTEMPTS'" ;;
+esac
+[ "$ATTEMPTS" -ge 1 ] || die "COSIGN_RETRY_ATTEMPTS must be >= 1, got '$ATTEMPTS'"
+
+case "$PER_ATTEMPT_TIMEOUT" in
+ ''|*[!0-9]*) die "COSIGN_RETRY_TIMEOUT must be a positive integer (seconds), got '$PER_ATTEMPT_TIMEOUT'" ;;
+esac
+[ "$PER_ATTEMPT_TIMEOUT" -ge 1 ] || die "COSIGN_RETRY_TIMEOUT must be >= 1, got '$PER_ATTEMPT_TIMEOUT'"
+
+# The backoff schedule must cover every retry the attempt budget permits.
+# Fail closed on a short schedule rather than silently retrying with no wait —
+# a zero-delay retry against a resetting CDN is the same request twice.
+read -r -a _delays <<< "$DELAYS"
+if [ "${#_delays[@]}" -lt "$((ATTEMPTS - 1))" ]; then
+ die "COSIGN_RETRY_DELAYS ('$DELAYS') declares ${#_delays[@]} delays but $ATTEMPTS attempts need $((ATTEMPTS - 1))"
+fi
+
+# ---------------------------------------------------------------------------
+# The transient vocabulary: sigstore/TUF/CDN and transport-level symptoms that
+# a second attempt can plausibly clear. Everything NOT matched here is treated
+# as a genuine refusal and fails immediately.
+#
+# Written as ONE list, consulted by ONE function, so the classifier the retry
+# loop uses is the same one the harness drives. The harness supplies its own
+# independently written failure texts (the real 2026-08-23 message among them)
+# rather than iterating this list — a list checked against itself is blind.
+# ---------------------------------------------------------------------------
+COSIGN_TRANSIENT_PATTERNS='tuf: failed to download
+error updating to TUF remote mirror
+connection reset by peer
+connection refused
+broken pipe
+i/o timeout
+TLS handshake timeout
+context deadline exceeded
+no such host
+temporary failure in name resolution
+unexpected EOF
+502 Bad Gateway
+503 Service Unavailable
+504 Gateway Time-?out
+429 Too Many Requests
+StatusCode: 5[0-9][0-9]'
+
+# is_transient <file> — true when the captured cosign output carries a known
+# transient signature. Exported behaviour: the harness calls this via the
+# script itself, never a copy.
+is_transient() {
+ printf '%s\n' "$COSIGN_TRANSIENT_PATTERNS" | grep -q '[^[:space:]]' || return 1
+ grep -qiE -f <(printf '%s\n' "$COSIGN_TRANSIENT_PATTERNS") "$1"
+}
+
+# The output files cosign was told to write, derived from the ACTUAL argument
+# list rather than a second hardcoded naming convention. Any --output-<x> flag
+# counts, so a new cosign output flag is covered without editing this script.
+outputs_from_args() {
+ local prev='' a
+ for a in "$@"; do
+ case "$prev" in --output-*) printf '%s\n' "$a" ;; esac
+ case "$a" in --output-*=*) printf '%s\n' "${a#*=}" ;; esac
+ prev="$a"
+ done
+}
+
+# run_cosign <logfile> <cosign args...> — one attempt, with its own wall-clock
+# cap enforced in bash. Deliberately NOT coreutils `timeout`: that is absent on
+# macOS, which would leave the cap's behaviour unverifiable for anyone running
+# the harness locally, and an untestable branch is how a cap stops being real.
+# Returns cosign's exit status, or 124 when the attempt was killed at the cap
+# (the same code `timeout` uses, so the caller reads identically).
+run_cosign() {
+ local log="$1"; shift
+ "$COSIGN_BIN" "$@" >"$log" 2>&1 &
+ local pid=$! waited=0
+ while kill -0 "$pid" 2>/dev/null; do
+ if [ "$waited" -ge "$PER_ATTEMPT_TIMEOUT" ]; then
+ kill -TERM "$pid" 2>/dev/null
+ sleep 1
+ kill -KILL "$pid" 2>/dev/null
+ wait "$pid" 2>/dev/null
+ return 124
+ fi
+ sleep 1
+ waited=$((waited + 1))
+ done
+ wait "$pid"
+}
+
+LOG="$(mktemp)"
+# shellcheck disable=SC2317 # reached via the EXIT trap
+cleanup() { rm -f "$LOG"; }
+trap cleanup EXIT
+
+attempt=1
+while : ; do
+ printf 'cosign-retry: attempt %d/%d: %s %s\n' "$attempt" "$ATTEMPTS" "$COSIGN_BIN" "$*"
+ run_cosign "$LOG" "$@"
+ rc=$?
+ cat "$LOG"
+
+ if [ "$rc" -eq 0 ]; then
+ # Fail closed: a zero exit is a claim, the artifact is the evidence.
+ missing=''
+ while IFS= read -r out; do
+ [ -n "$out" ] || continue
+ [ -s "$out" ] || missing="$missing $out"
+ done < <(outputs_from_args "$@")
+ if [ -n "$missing" ]; then
+ printf 'cosign-retry: cosign exited 0 but produced no signature artifact:%s\n' "$missing" >&2
+ printf 'cosign-retry: refusing to report a signed binary on an absent signature (not retried — a zero exit with no output is not a network symptom)\n' >&2
+ exit 3
+ fi
+ printf 'cosign-retry: signed on attempt %d/%d\n' "$attempt" "$ATTEMPTS"
+ exit 0
+ fi
+
+ # Exit 124 is the per-attempt cap killing a hung attempt — a stalled network call by
+ # construction, so it is transient without needing to match any text.
+ if [ "$rc" -eq 124 ]; then
+ printf 'cosign-retry: attempt %d timed out after %ss (treated as transient)\n' "$attempt" "$PER_ATTEMPT_TIMEOUT" >&2
+ elif is_transient "$LOG"; then
+ printf 'cosign-retry: attempt %d failed (rc=%d) on a known transient sigstore/network signature\n' "$attempt" "$rc" >&2
+ else
+ printf 'cosign-retry: attempt %d failed (rc=%d) and the output matches no transient sigstore/network signature.\n' "$attempt" "$rc" >&2
+ printf 'cosign-retry: this is a genuine signing refusal, not a flake — NOT retrying.\n' >&2
+ exit "$rc"
+ fi
+
+ if [ "$attempt" -ge "$ATTEMPTS" ]; then
+ printf 'cosign-retry: exhausted %d attempts; the transient failure did not clear. Failing so the aggregate gate holds and no partially-signed release is published.\n' "$ATTEMPTS" >&2
+ exit "$rc"
+ fi
+
+ delay="${_delays[$((attempt - 1))]}"
+ printf 'cosign-retry: retrying in %ss\n' "$delay" >&2
+ sleep "$delay"
+ attempt=$((attempt + 1))
+done