fix(sync): watch the data-ingestors pin for contract-version drift (backend#2704)

.cursor/BUGBOT.md

@@ -152,6 +152,11 @@ Two things make this repo unusual and should shape every finding:
 - `test/integration/*` uses 30s–5min timeouts because it drives a real cluster — not the
  production timeout convention.
 - `mutation.yml` and `head-drift-canary.yml` are advisory and never gate a merge.
+ `pin-version-drift.yml` is scheduled-only (no `pull_request` trigger): on data-ingestors
+ contract-version drift (or any cannot-evaluate) it opens/updates a `pin-version-drift`-labelled
+ tracking issue AND fails closed — a red scheduled run plus a filed issue is the intended signal.
+ It still never gates a PR merge; gating PRs on live upstream is the backend#1009 failure it
+ avoids (backend#2704).
 - No `vendor/` directory — the module cache is used on purpose.
 - `// style-guard: allow` is a defined escape hatch but is currently used nowhere; if one
  appears, it is a novel exception worth scrutiny rather than an accepted pattern.

.github/workflows/build.yml

@@ -87,6 +87,10 @@ jobs:
  bash -n scripts/tests/install-verify.sh
  shellcheck --shell=bash --severity=error scripts/tests/install-ps1-verify.sh
  bash -n scripts/tests/install-ps1-verify.sh
+ shellcheck --shell=bash --severity=error scripts/check-pin-version.sh
+ shellcheck --shell=bash --severity=error scripts/tests/pin-version-verify.sh
+ bash -n scripts/check-pin-version.sh
+ bash -n scripts/tests/pin-version-verify.sh
  # format.sh's own fail-closed properties. Formatters are stubbed, so this is
  # hermetic and needs no Go toolchain — which is why it lives in this job
  # rather than Lint. It exists because the first cut of format.sh reported
@@ -108,6 +112,13 @@ jobs:
  # since "cannot tell" is not evidence that verification is mandatory.
  - name: Verification harness — Windows (mandatory cosign / fail-closed)
  run: bash scripts/tests/install-ps1-verify.sh
+ # check-pin-version.sh's fail-closed properties: contract-version DRIFT
+ # reddens (exit 1) and every "cannot evaluate" reddens (exit 2). Fetches
+ # are file:// URLs and the pin/branch are injected, so this is hermetic
+ # and needs no network — the live pin-vs-HEAD comparison is the scheduled
+ # pin-version-drift.yml, which must never gate a PR (backend#2704 / #1009).
+ - name: Pin-version watcher harness (drift reddens / fail-closed)
+ run: bash scripts/tests/pin-version-verify.sh
 
  test:
  timeout-minutes: 15

.github/workflows/pin-version-drift.yml

@@ -0,0 +1,126 @@
+name: Contract-version drift (pin freshness)
+
+# Watches the data-ingestors pin ITSELF for contract-version staleness
+# (backend#2704). scripts/sync-schema.sh --check (build.yml) answers "does my
+# vendored copy match the SHA I pinned?"; it never answers "is the SHA I pinned
+# still current?" So the pin sat a whole contract VERSION behind (layout v2 vs
+# data-ingestors@develop v3) while every drift gate stayed green.
+#
+# This job resolves scripts/.data-ingestors-ref, resolves data-ingestors'
+# current default branch, and compares the CONTRACT VERSION — the top-level
+# "version" in tracebloc_ingestor/schema/layout.v1.json — at the pin vs at HEAD.
+# Version, NOT commit distance: 180 unrelated upstream commits are fine, a
+# version bump is the finding (scripts/check-pin-version.sh).
+#
+# SCHEDULED, never a PR gate. Gating open PRs on live upstream is precisely the
+# backend#1009 failure this repo removed by pinning — an unrelated upstream
+# bump would red every open PR. So there is no `pull_request` trigger here; the
+# checker's own logic is guarded on PRs by its hermetic unit test
+# (scripts/tests/pin-version-verify.sh, run in build.yml), which touches no
+# network. The pinned PR comparison (sync-schema.sh --check) is left unchanged.
+#
+# FAIL CLOSED, AND it reaches a person. On version drift (checker exit 1) or any
+# "cannot evaluate" — absent/malformed pin, moved path, unreachable raw URL,
+# non-JSON, missing "version", unparseable sync-schema.sh (exit 2) — the job
+# opens or updates ONE tracking issue (label `pin-version-drift`) AND then fails.
+# The issue is why a red run isn't the *only* signal: a scheduled red that no one
+# is assigned to would quietly become this workflow's normal state (LukasWodka on
+# cli#595). The finding lands where a person sees it, the way head-drift-canary.yml
+# does; the job also reds, so "cannot tell" is never a green pass.
+#
+# data-ingestors is public, so github.token suffices for the `gh api`
+# default-branch lookup and the raw fetches. If it ever goes private, add a
+# read token (CROSS_REPO_READ_TOKEN) the way head-drift-canary.yml does.
+
+on:
+ schedule:
+ - cron: "30 6 * * 1" # weekly Monday 06:30 UTC — after head-drift-canary (06:00)
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ issues: write
+
+concurrency:
+ group: pin-version-drift
+ cancel-in-progress: false
+
+jobs:
+ contract-version:
+ timeout-minutes: 10
+ name: Pinned contract version vs HEAD (fail-closed)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+ - name: Compare pinned contract version against data-ingestors HEAD
+ id: check
+ # Capture the checker's exit WITHOUT failing the step, so the reporting
+ # step can run before the job reds. GH_TOKEN lets `gh api` resolve the
+ # current default branch. python3 and curl are preinstalled on ubuntu.
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ rc=0
+ ./scripts/check-pin-version.sh > check-out.txt 2>&1 || rc=$?
+ cat check-out.txt
+ echo "rc=$rc" >> "$GITHUB_OUTPUT"
+ case "$rc" in
+ 0) echo "state=in_sync" >> "$GITHUB_OUTPUT" ;;
+ 1) echo "state=drift" >> "$GITHUB_OUTPUT" ;;
+ *) echo "state=blocked" >> "$GITHUB_OUTPUT" ;;
+ esac
+
+ - name: Open or update the pin-version-drift tracking issue
+ if: steps.check.outputs.rc != '0'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPO: ${{ github.repository }}
+ STATE: ${{ steps.check.outputs.state }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ if [ "$STATE" = "drift" ]; then
+ title="data-ingestors contract version has moved past the CLI's pin"
+ headline="The pinned data-ingestors layout contract is a VERSION behind HEAD."
+ else
+ title="pin-version-drift watcher could not evaluate the pin (fail-closed)"
+ headline="The watcher could not evaluate the pin and fail-closed — this is a finding, not agreement."
+ fi
+ # shellcheck disable=SC2016 # printf formats carry markdown backticks, not shell expansions
+ {
+ printf '<!-- pin-version-drift -->\n'
+ printf '**%s**\n\n' "$headline"
+ printf '**Scheduled run:** %s\n\n' "$RUN_URL"
+ printf '```\n'
+ cat check-out.txt
+ printf '\n```\n\n'
+ if [ "$STATE" = "drift" ]; then
+ printf '**Fix (cli#286 shape):** bump `scripts/.data-ingestors-ref`, run `scripts/sync-schema.sh`, and commit the regenerated `internal/schema/*` in one deliberate PR. Close this issue when that lands.\n'
+ else
+ printf 'A bad/absent pin, a moved path, an unreachable raw URL, a non-JSON body, a missing `"version"`, or an unparseable `sync-schema.sh` all land here — "cannot tell" is never a green pass. Fix the cause (see the run log), then re-run this workflow. Close this issue once a run is clean.\n'
+ fi
+ } > body.md
+ gh label create pin-version-drift --repo "$REPO" --force \
+ --description "auto-filed by pin-version-drift.yml (backend#2704)" --color D93F0B
+ # Keyed by the label so there is at most ONE open issue; refresh its
+ # title each run so it reflects the current failure mode.
+ existing="$(gh issue list --repo "$REPO" --label pin-version-drift --state open \
+ --json number --jq '.[0].number // empty')"
+ if [ -n "$existing" ]; then
+ echo "updating existing tracking issue #$existing"
+ gh issue edit "$existing" --repo "$REPO" --title "$title"
+ gh issue comment "$existing" --repo "$REPO" --body-file body.md
+ else
+ echo "opening new tracking issue"
+ gh issue create --repo "$REPO" --title "$title" --body-file body.md --label pin-version-drift
+ fi
+
+ - name: Fail closed on drift or cannot-evaluate
+ if: steps.check.outputs.rc != '0'
+ run: |
+ echo "::error::pin-version watcher exit ${{ steps.check.outputs.rc }} (${{ steps.check.outputs.state }}) — a tracking issue was opened/updated (label: pin-version-drift)."
+ exit 1
+
+ - name: All clear
+ if: steps.check.outputs.rc == '0'
+ run: echo "::notice::Pinned contract version matches data-ingestors HEAD."

scripts/check-pin-version.sh

@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# =============================================================================
+# check-pin-version.sh — is the pinned data-ingestors contract still the
+# CURRENT contract version? (backend#2704)
+#
+# scripts/sync-schema.sh --check answers one question well: "does my vendored
+# copy match the SHA I pinned?" It never answers the second: "is the SHA I
+# pinned still current?" So the pin can sit six weeks / a whole contract
+# VERSION behind while every drift gate stays green — which is exactly what
+# happened (pin on layout contract v2, data-ingestors@develop on v3).
+#
+# The pin is deliberate and must stay (backend#1009: a floating ref reds every
+# open CLI PR on unrelated upstream commits). This script does NOT float it. It
+# is the watcher for the pin ITSELF: resolve the pin, resolve data-ingestors'
+# current default branch, fetch the dataset-layout contract at BOTH refs, and
+# compare the CONTRACT VERSION — the top-level "version" string in
+# tracebloc_ingestor/schema/layout.v1.json.
+#
+# VERSION, not commit distance. 180 unrelated upstream commits that never touch
+# the version are fine and stay green; a single version bump is the finding.
+# This is why it is a SCHEDULED job, never a PR gate: gating PRs on live
+# upstream is the backend#1009 failure this repo already removed.
+#
+# FAIL CLOSED. "Cannot tell" is a finding, never a pass — the guard already has
+# one shape of "cannot tell reads as fine" (the pinned check going stale) and
+# must not gain a second. An absent/malformed pin, a moved path, an unreachable
+# raw URL, a non-JSON body, or a missing "version" all exit non-zero.
+#
+# Exit codes:
+# 0 IN SYNC — the pin is on the current contract version.
+# 1 DRIFT — the pin's contract version differs from HEAD's. Re-pin
+# (scripts/.data-ingestors-ref) and re-run scripts/sync-schema.sh.
+# 2 FAIL CLOSED — could not evaluate (bad/absent pin, moved path, unreachable
+# URL, non-JSON, missing version, or an unparseable
+# sync-schema.sh). Never reported as agreement.
+#
+# Env knobs (defaults are the production values; overrides exist for the
+# hermetic test harness, scripts/tests/pin-version-verify.sh):
+# REF_FILE pin file (default: scripts/.data-ingestors-ref)
+# SYNC_SCHEMA_SH source of the vendored layout path
+# (default: scripts/sync-schema.sh)
+# DATA_INGESTORS_REF override the pin ref (default: from REF_FILE)
+# DATA_INGESTORS_DEFAULT_BRANCH override the upstream default branch
+# (default: resolved via `gh api`)
+# PIN_LAYOUT_URL / HEAD_LAYOUT_URL full URL overrides for the two fetches
+# (default: built from the refs above)
+# =============================================================================
+set -uo pipefail
+
+readonly UPSTREAM_BASE="https://raw.githubusercontent.com/tracebloc/data-ingestors"
+
+_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REF_FILE="${REF_FILE:-${_here}/.data-ingestors-ref}"
+# sync-schema.sh is the single source of truth for the vendored layout path;
+# see derive_layout_subpath. SYNC_SCHEMA_SH lets the test harness point at a
+# fixture.
+SYNC_SCHEMA_SH="${SYNC_SCHEMA_SH:-${_here}/sync-schema.sh}"
+
+die_closed() { echo "FAIL CLOSED: $*" >&2; exit 2; }
+
+# A ref is interpolated into a raw.githubusercontent URL, so validate its shape
+# before use — same guard as sync-schema.sh / chart-drift.yml: alnum start, then
+# alnum . _ - / and no ".." component. Blocks path-traversal / extra segments.
+valid_ref_shape() {
+ grep -qE '^[A-Za-z0-9][A-Za-z0-9._/-]*$' <<<"$1" && ! grep -q '\.\.' <<<"$1"
+}
+
+# Derive the upstream layout path (tracebloc_ingestor/schema/layout.v1.json)
+# from sync-schema.sh instead of holding a second copy: sync-schema.sh is the
+# single source of truth for the vendored path, so a moved schema dir there
+# changes both in one place, and the old "keep in lockstep" comment becomes an
+# actual check (LukasWodka on cli#595; pin-version-verify.sh asserts the derived
+# value). Returns non-zero — the caller fail-closes — if either declaration it
+# parses is gone. It parses these two sync-schema.sh lines:
+# UPSTREAM_BASE="...:${DATA_INGESTORS_REF}/<dir>"
+# "${UPSTREAM_BASE}/<file>|internal/schema/layout.v1.json"
+derive_layout_subpath() {
+ [[ -f "$SYNC_SCHEMA_SH" ]] || { echo " no sync-schema.sh at $SYNC_SCHEMA_SH" >&2; return 1; }
+ local dir file
+ dir="$(grep -oE 'DATA_INGESTORS_REF\}/[A-Za-z0-9._/-]+' "$SYNC_SCHEMA_SH" | head -1 | sed -E 's#^DATA_INGESTORS_REF\}/##')"
+ file="$(grep -oE 'UPSTREAM_BASE\}/[A-Za-z0-9._-]+\.json\|internal/schema/layout\.v1\.json' "$SYNC_SCHEMA_SH" | head -1 | sed -E 's#^UPSTREAM_BASE\}/##; s#\|.*$##')"
+ [[ -n "$dir" && -n "$file" ]] || { echo " could not parse the layout path from $SYNC_SCHEMA_SH" >&2; return 1; }
+ printf '%s/%s' "$dir" "$file"
+}
+
+# Resolve the pin: first non-comment, non-blank line of REF_FILE (the rule
+# sync-schema.sh uses), overridable via DATA_INGESTORS_REF.
+resolve_pin() {
+ if [[ -n "${DATA_INGESTORS_REF:-}" ]]; then
+ printf '%s' "$DATA_INGESTORS_REF"; return 0
+ fi
+ [[ -f "$REF_FILE" ]] || return 1
+ grep -vE '^[[:space:]]*(#|$)' "$REF_FILE" 2>/dev/null | head -1 | tr -d '[:space:]'
+}
+
+# Resolve data-ingestors' CURRENT default branch, overridable for tests. Uses
+# `gh api` (needs GH_TOKEN in CI); a public repo's metadata reads with the
+# default workflow token.
+resolve_default_branch() {
+ if [[ -n "${DATA_INGESTORS_DEFAULT_BRANCH:-}" ]]; then
+ printf '%s' "$DATA_INGESTORS_DEFAULT_BRANCH"; return 0
+ fi
+ local b
+ b="$(gh api repos/tracebloc/data-ingestors --jq .default_branch 2>/dev/null)"
+ # gh emits nothing on failure and the literal "null" when the field is absent
+ # (a 200 with an unexpected body). Treat both as unresolved so main's -n check
+ # fail-closes on the real cause instead of curling a `.../null/...` URL.
+ [[ -n "$b" && "$b" != "null" ]] || return 1
+ printf '%s' "$b"
+}
+
+# fetch_version <url> <label> — echo the top-level "version" string on success;
+# any failure (fetch, non-JSON, missing/empty/non-scalar version) returns
+# non-zero so the caller fail-closes. The body is captured into a variable, not
+# a temp file: fetch_version is called as `$(fetch_version ...)`, i.e. in a
+# subshell where an EXIT trap is reset and a `_tmpfiles+=` would be invisible to
+# the parent — so a temp file here would leak. No temp file, nothing to clean.
+fetch_version() {
+ local url="$1" label="$2" body ver
+ # -f: HTTP errors (404 moved-path) are failures; time bounds keep a hung
+ # endpoint from wedging the run. Mirrors sync-schema.sh's curl.
+ if ! body="$(curl -fsSL --tlsv1.2 --connect-timeout 10 --max-time 60 "$url")"; then
+ echo " could not fetch the $label contract: $url" >&2
+ return 1
+ fi
+ # python3 exits non-zero on a non-JSON body or a missing/empty/non-scalar
+ # "version"; pipefail carries that exit to the `if !`.
+ # shellcheck disable=SC2016 # the $-expressions are Python, not shell
+ if ! ver="$(printf '%s' "$body" | python3 -c '
+import json, sys
+try:
+ data = json.load(sys.stdin)
+except Exception:
+ sys.exit(1)
+v = data.get("version")
+if not isinstance(v, (str, int)) or isinstance(v, bool) or str(v).strip() == "":
+ sys.exit(1)
+print(str(v).strip())
+')"; then
+ echo " the $label contract is not JSON, or has no top-level \"version\": $url" >&2
+ return 1
+ fi
+ printf '%s' "$ver"
+}
+
+main() {
+ local pin default_branch pin_url head_url pin_ver head_ver layout_subpath=""
+
+ pin="$(resolve_pin)"
+ [[ -n "$pin" ]] || die_closed "no ref in ${REF_FILE} (first non-comment line must be a commit SHA)"
+ valid_ref_shape "$pin" || die_closed "invalid pin ref shape: '$pin'"
+
+ # Derive the layout path only when we actually build a URL from it — the test
+ # harness overrides both URLs and then needs neither sync-schema.sh nor a
+ # network fetch.
+ if [[ -z "${PIN_LAYOUT_URL:-}" || -z "${HEAD_LAYOUT_URL:-}" ]]; then
+ layout_subpath="$(derive_layout_subpath)" \
+ || die_closed "could not derive the layout path from ${SYNC_SCHEMA_SH} (its format changed?)"
+ fi
+
+ # Build the pin URL (overridable). The pin comes from the ref file above.
+ pin_url="${PIN_LAYOUT_URL:-${UPSTREAM_BASE}/${pin}/${layout_subpath}}"
+
+ # Build the HEAD URL (overridable). Only resolve the default branch when the
+ # URL isn't overridden, so the test harness needs no `gh` and no network.
+ if [[ -n "${HEAD_LAYOUT_URL:-}" ]]; then
+ head_url="$HEAD_LAYOUT_URL"
+ default_branch="(HEAD_LAYOUT_URL override)"
+ else
+ default_branch="$(resolve_default_branch)"
+ [[ -n "$default_branch" ]] || die_closed "could not resolve data-ingestors' default branch (gh api)"
+ valid_ref_shape "$default_branch" || die_closed "invalid default-branch shape: '$default_branch'"
+ head_url="${UPSTREAM_BASE}/${default_branch}/${layout_subpath}"
+ fi
+
+ pin_ver="$(fetch_version "$pin_url" "pinned")" || die_closed "could not read the pinned contract version"
+ head_ver="$(fetch_version "$head_url" "HEAD")" || die_closed "could not read the HEAD contract version"
+
+ echo "data-ingestors dataset-layout contract version"
+ echo " pin ${pin} -> v${pin_ver}"
+ echo " default branch ${default_branch} -> v${head_ver}"
+
+ if [[ "$pin_ver" == "$head_ver" ]]; then
+ echo "IN SYNC: the pin is on the current contract version (v${pin_ver})."
+ return 0
+ fi
+
+ echo "DRIFT: the pinned contract is v${pin_ver}, but data-ingestors HEAD is v${head_ver}." >&2
+ echo "The CLI's vendored layout/sidecar/help are derived from a stale contract version." >&2
+ echo "Fix: bump scripts/.data-ingestors-ref, run scripts/sync-schema.sh, commit the regenerated files (cli#286 shape)." >&2
+ return 1
+}
+
+# Run main only when executed directly, not when sourced — so the test harness
+# can source this file and call derive_layout_subpath in isolation.
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+ main "$@"
+fi