fix(release): self-heal a lost CDN rebuild and surface tap push errors

.changeset/reliable-release-channel.md

@@ -0,0 +1,5 @@
+---
+"@pymodel/pythinker-code": patch
+---
+
+Keep releases visible in the update channel when a CDN rebuild request is temporarily lost.

.changeset/use-app-token-for-brew-tap.md

@@ -0,0 +1,5 @@
+---
+"@pymodel/pythinker-code": patch
+---
+
+Use a scoped GitHub App token for Homebrew tap updates.

.github/workflows/release.yml

@@ -264,11 +264,14 @@ jobs:
  # nothing — an earlier version of this job was deleted because a
  # curl exit-28 timeout failed the 0.5.0 release. verify-cdn-release
  # polls the manifest and is the gate that fails loudly.
+ # On 0.18.0, one connect consumed the full 60-second budget. A short
+ # connect timeout turns the same wall-clock budget into more attempts
+ # during an outage instead of waiting on connections never made.
  status=$(curl -sS -o /dev/stderr -w '%{http_code}' -X POST "$WEBHOOK" \
  -H 'Content-Type: application/json' \
  -H 'X-GitHub-Event: push' \
  -d '{"ref":"refs/heads/main"}' \
- --retry 3 --retry-all-errors --retry-delay 10 --max-time 60) || status=000
+ --connect-timeout 15 --max-time 45 --retry 5 --retry-all-errors --retry-delay 15) || status=000
  case "$status" in
  2*) echo "CDN redeploy triggered (HTTP $status)." ;;
  *) echo "::warning::CDN redeploy webhook returned HTTP $status — verify-cdn-release will catch a stale CDN." ;;
@@ -282,7 +285,7 @@ jobs:
  # publish hid the one case where the version and the published artifacts
  # diverge — and every client polled the CDN for a release that never existed.
  verify-cdn-release:
- timeout-minutes: 15
+ timeout-minutes: 20
  name: Verify release consistency
  needs:
  - release
@@ -303,6 +306,8 @@ jobs:
  node-version-file: .nvmrc
 
  - name: Verify release consistency
+ env:
+ DOKPLOY_CDN_DEPLOY_WEBHOOK: ${{ secrets.DOKPLOY_CDN_DEPLOY_WEBHOOK }}
  run: node scripts/release/verify-release-consistency.mjs
 
  update-brew-tap:
@@ -320,10 +325,24 @@ jobs:
  with:
  node-version-file: .nvmrc
 
+ # Any permission-* input switches the token from inheriting every
+ # permission the App installation holds to exactly the ones listed here.
+ # Cloning and pushing the tap needs contents and nothing else.
+ - name: Mint tap token
+ id: tap-token
+ uses: actions/create-github-app-token@v2
+ with:
+ app-id: ${{ vars.RELEASE_BOT_APP_ID }}
+ private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }}
+ owner: PyModel
+ repositories: homebrew-tap
+ permission-contents: write
+
  - name: Bump formula
  env:
- TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+ TAP_GITHUB_TOKEN: ${{ steps.tap-token.outputs.token }}
  run: |
+ # The token comes from the App installation, not a PAT.
  if [ -z "$TAP_GITHUB_TOKEN" ]; then
  echo "TAP_GITHUB_TOKEN secret not set — skipping tap update" >&2
  exit 0

apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts

@@ -111,6 +111,125 @@ describe('pollCdnUntilCaughtUp', () => {
  expect(result).toMatchObject({ ok: true, reason: 'match', attempts: 3 });
  });
 
+ it('re-triggers on the configured cadence while the CDN is behind', async () => {
+ const { now, sleep } = fakeClock();
+ let retriggerCalls = 0;
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([
+ () => manifest('0.12.0'),
+ () => manifest('0.12.0'),
+ () => manifest('0.12.0'),
+ () => manifest('0.12.0'),
+ () => manifest('0.12.0'),
+ () => manifest('0.12.0'),
+ () => manifest('0.13.0'),
+ ]),
+ retrigger: async () => {
+ retriggerCalls += 1;
+ },
+ retriggerEveryAttempts: 3,
+ });
+
+ expect(retriggerCalls).toBe(2);
+ expect(result).toMatchObject({ ok: true, attempts: 7, retriggers: 2 });
+ });
+
+ it('does not re-trigger when the first attempt matches', async () => {
+ const { now, sleep } = fakeClock();
+ let retriggerCalls = 0;
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([() => manifest('0.13.0')]),
+ retrigger: async () => {
+ retriggerCalls += 1;
+ },
+ retriggerEveryAttempts: 1,
+ });
+
+ expect(retriggerCalls).toBe(0);
+ expect(result.retriggers).toBe(0);
+ });
+
+ it('does not re-trigger when the CDN is ahead', async () => {
+ const { now, sleep } = fakeClock();
+ let retriggerCalls = 0;
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([() => manifest('0.14.0')]),
+ retrigger: async () => {
+ retriggerCalls += 1;
+ },
+ retriggerEveryAttempts: 1,
+ });
+
+ expect(retriggerCalls).toBe(0);
+ expect(result).toMatchObject({ reason: 'ahead', retriggers: 0 });
+ });
+
+ it('keeps polling when a re-trigger throws', async () => {
+ const { now, sleep } = fakeClock();
+ let retriggerCalls = 0;
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([() => manifest('0.12.0'), () => manifest('0.13.0')]),
+ retrigger: async () => {
+ retriggerCalls += 1;
+ throw new Error('trigger failed');
+ },
+ retriggerEveryAttempts: 1,
+ });
+
+ expect(retriggerCalls).toBe(1);
+ expect(result).toMatchObject({ ok: true, attempts: 2, retriggers: 1 });
+ });
+
+ it('re-triggers while the CDN is unreachable', async () => {
+ const { now, sleep } = fakeClock();
+ let retriggerCalls = 0;
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([
+ () => {
+ throw new Error('ECONNREFUSED');
+ },
+ () => manifest('0.13.0'),
+ ]),
+ retrigger: async () => {
+ retriggerCalls += 1;
+ },
+ retriggerEveryAttempts: 1,
+ });
+
+ expect(retriggerCalls).toBe(1);
+ expect(result).toMatchObject({ ok: true, retriggers: 1 });
+ });
+
+ it('reports re-trigger attempts when the budget expires', async () => {
+ const { now, sleep } = fakeClock();
+ const result = await pollCdnUntilCaughtUp({
+ ...base,
+ budgetMs: 45_000,
+ now,
+ sleep,
+ fetchImpl: scriptedFetch([() => manifest('0.12.0')]),
+ retrigger: async () => {},
+ retriggerEveryAttempts: 1,
+ });
+
+ expect(result).toMatchObject({ reason: 'timeout', attempts: 3, retriggers: 2 });
+ });
+
  it('treats an unreachable CDN as lag rather than a failure', async () => {
  const { now, sleep } = fakeClock();
  const result = await pollCdnUntilCaughtUp({

scripts/release/cdn-consistency.mjs

@@ -67,12 +67,16 @@ async function readCdnVersion(fetchImpl, url) {
  * half-written, and both that and plain lag resolve by waiting, so only the
  * budget decides. 'ahead' returns at once — more waiting cannot fix a manifest
  * that names a release npm does not have.
+ *
+ * A periodic re-trigger heals a lost deploy request. This poll is the only
+ * pipeline stage that both knows the CDN is still behind and is still running.
  */
 export async function pollCdnUntilCaughtUp(options) {
- const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs } = options;
+ const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs, retrigger, retriggerEveryAttempts } = options;
  const deadline = now() + budgetMs;
  let cdnVersion = null;
  let attempts = 0;
+ let retriggers = 0;
 
  for (;;) {
  attempts += 1;
@@ -87,12 +91,25 @@ export async function pollCdnUntilCaughtUp(options) {
  // Deliberately swallowed: an unreachable CDN is lag, not a gate failure.
  }
 
- if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts };
- if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts };
+ if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts, retriggers };
+ if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts, retriggers };
 
  // Stop before a sleep that would run past the budget rather than after it.
  if (now() + intervalMs >= deadline) {
- return { ok: false, reason: 'timeout', cdnVersion, attempts };
+ return { ok: false, reason: 'timeout', cdnVersion, attempts, retriggers };
+ }
+ if (
+ typeof retrigger === 'function' &&
+ Number.isInteger(retriggerEveryAttempts) &&
+ retriggerEveryAttempts > 0 &&
+ attempts % retriggerEveryAttempts === 0
+ ) {
+ retriggers += 1;
+ try {
+ await retrigger();
+ } catch {
+ // Deliberately swallowed: a failed trigger is lag, not a gate failure.
+ }
  }
  await sleep(intervalMs);
  }

scripts/release/update-brew-formula.mjs

@@ -4,6 +4,11 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
+function redactGitOutput(value, token) {
+ const redacted = String(value ?? '').replaceAll(/\/\/x-access-token:[^@\s]*@/gu, '//***@');
+ return token.length >= 8 ? redacted.replaceAll(token, '***') : redacted;
+}
+
 async function main() {
  const packageJson = JSON.parse(readFileSync(new URL('../../apps/pythinker-code/package.json', import.meta.url), 'utf8'));
  const version = packageJson.version;
@@ -22,10 +27,13 @@ async function main() {
  try {
  try {
  execFileSync('git', ['clone', `https://x-access-token:${token}@github.com/PyModel/homebrew-tap.git`, tapDir], {
- stdio: 'ignore',
+ stdio: 'pipe',
  });
- } catch {
- throw new Error('Failed to clone Homebrew tap');
+ } catch (error) {
+ const stderr = redactGitOutput(error.stderr, token).trim();
+ const stdout = redactGitOutput(error.stdout, token).trim();
+ const message = redactGitOutput(error.message, token).trim();
+ throw new Error(`Failed to clone Homebrew tap: ${stderr || stdout || message}`, { cause: error });
  }
 
  const formulaPath = join(tapDir, 'Formula/pythinker-code.rb');
@@ -62,9 +70,12 @@ async function main() {
  { cwd: tapDir, stdio: 'inherit' },
  );
  try {
- execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'ignore' });
- } catch {
- throw new Error('Failed to push Homebrew tap');
+ execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'pipe' });
+ } catch (error) {
+ const stderr = redactGitOutput(error.stderr, token).trim();
+ const stdout = redactGitOutput(error.stdout, token).trim();
+ const message = redactGitOutput(error.message, token).trim();
+ throw new Error(`Failed to push Homebrew tap: ${stderr || stdout || message}`, { cause: error });
  }
  } finally {
  rmSync(tapDir, { recursive: true, force: true });

scripts/release/verify-release-consistency.mjs

@@ -8,10 +8,9 @@ const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[
 
 const CDN_MANIFEST_URL = 'https://code.pythinker.com/pythinker-code/latest.json';
 
-// A Dokploy rebuild serves the new manifest in roughly two minutes. The budget
-// stays well under the job's own timeout-minutes so a stale CDN is reported
-// here rather than killed by the runner.
-const CDN_POLL_BUDGET_MS = 600_000;
+// The budget covers detecting a lost trigger and completing a fresh rebuild,
+// while staying under the job timeout so this gate can report a stale CDN.
+const CDN_POLL_BUDGET_MS = 900_000;
 const CDN_POLL_INTERVAL_MS = 15_000;
 
 function fail(reason) {
@@ -59,6 +58,41 @@ try {
 }
 if (!gitTags.trim().split('\n').includes(releaseTag)) fail(`missing git tag ${releaseTag}`);
 
+const webhook = process.env.DOKPLOY_CDN_DEPLOY_WEBHOOK;
+let retrigger;
+if (typeof webhook === 'string' && webhook.length > 0) {
+ let isUsable;
+ try {
+ const url = new URL(webhook);
+ isUsable = url.protocol === 'https:' && url.host.length > 0;
+ } catch {
+ isUsable = false;
+ }
+ if (isUsable) {
+ retrigger = async () => {
+ try {
+ const response = await fetch(webhook, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'X-GitHub-Event': 'push',
+ },
+ body: '{"ref":"refs/heads/main"}',
+ signal: AbortSignal.timeout(30_000),
+ });
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ console.log(`CDN rebuild request returned HTTP ${response.status}`);
+ } catch (error) {
+ const message = error instanceof Error ? error.message.replaceAll(webhook, '***') : 'unknown error';
+ console.error(`CDN rebuild request failed: ${message}`);
+ throw error;
+ }
+ };
+ } else {
+ console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled');
+ }
+}
+
 // The CDN manifest is what every installed client polls for updates. A version
 // it advertises that npm does not have sends all of them into an install that
 // cannot succeed; a version it never catches up to hides the release entirely.
@@ -75,6 +109,8 @@ const cdnPoll = await pollCdnUntilCaughtUp({
  npmLatest: distTags.latest,
  budgetMs: CDN_POLL_BUDGET_MS,
  intervalMs: CDN_POLL_INTERVAL_MS,
+ retrigger,
+ retriggerEveryAttempts: 8,
 });
 
 if (cdnPoll.reason === 'ahead') {
@@ -87,11 +123,15 @@ if (!cdnPoll.ok) {
  fail(
  `CDN never caught up with npm within ${CDN_POLL_BUDGET_MS / 1000}s ` +
  `(cdn=${cdnPoll.cdnVersion ?? 'unreachable'} latest=${distTags.latest}, ` +
- `${cdnPoll.attempts} attempts) — every installed client polls this manifest, ` +
+ `${cdnPoll.attempts} attempt(s), ${cdnPoll.retriggers} rebuild request(s)) — ` +
+ 'every installed client polls this manifest, ' +
  'so the release stays invisible until the site rebuilds',
  );
 }
 
-console.log(`CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s)`);
+console.log(
+ `CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s), ` +
+ `${cdnPoll.retriggers} rebuild request(s)`,
+);
 
 console.log(`consistency OK: latest=${distTags.latest} beta=${distTags.beta ?? '-'} dev=${distTags.dev ?? '-'}`);