From 34812a8f222e3866883a57fc861528b83093e0e8 Mon Sep 17 00:00:00 2001 From: Michael Buluma <1452922+buluma@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:47:32 +0300 Subject: [PATCH 1/4] ci: throwaway probe job for the ProviderRegistry real-spawn hang theory Isolates ProviderRegistry.test.ts and ProviderInstanceRegistryLive.test.ts (the two suites that wire the real OpenCodeRuntimeLive and genuinely spawn subprocesses) on their own runner, to check whether PR #48's opencodeRuntime.ts change (detached spawn + process-group kill finalizer) is what's hanging "Test Server 2" on every run since. Delete once answered. --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8bd0991..9fa10e30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,6 +117,46 @@ jobs: - name: Test run: vp run --filter helmcode test --shard ${{ matrix.shard }}/${{ strategy.job-total }} + # THROWAWAY: isolates the two real-subprocess-spawning ProviderRegistry + # suites to find out whether PR #48's opencodeRuntime.ts change (detached + # spawn + process-group kill finalizer) is what's hanging "Test Server 2" + # on every run since. Delete this job once the probe answers that. + probe_opencode_hang: + name: Probe OpenCode Hang + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Probe + run: | + set -o pipefail + vp run --filter helmcode test \ + apps/server/src/provider/Layers/ProviderRegistry.test.ts \ + apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts \ + -- --reporter=verbose 2>&1 | tee /tmp/probe.log + + - name: Upload probe log + if: always() + uses: actions/upload-artifact@v4 + with: + name: probe-log + path: /tmp/probe.log + if-no-files-found: ignore + # Split out of Check and Test: both paid a Rust toolchain install for checks # that take a couple seconds, on the critical path of every PR. rust: From 1a48f49849abf5cf0bd6f9fe9408a09010b3e669 Mon Sep 17 00:00:00 2001 From: Michael Buluma <1452922+buluma@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:53:24 +0300 Subject: [PATCH 2/4] ci: fix probe test paths to be relative to apps/server --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fa10e30..53c3cf88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,8 +145,8 @@ jobs: run: | set -o pipefail vp run --filter helmcode test \ - apps/server/src/provider/Layers/ProviderRegistry.test.ts \ - apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts \ + src/provider/Layers/ProviderRegistry.test.ts \ + src/provider/Layers/ProviderInstanceRegistryLive.test.ts \ -- --reporter=verbose 2>&1 | tee /tmp/probe.log - name: Upload probe log From 37877336eb1918cb0aeeeefa7de1fceb7c7fdb35 Mon Sep 17 00:00:00 2001 From: Michael Buluma <1452922+buluma@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:23:50 +0300 Subject: [PATCH 3/4] ci: split probe into one job per suspect file, capture partial log on hang --- .github/workflows/ci.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53c3cf88..ffb0dcb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,14 +117,21 @@ jobs: - name: Test run: vp run --filter helmcode test --shard ${{ matrix.shard }}/${{ strategy.job-total }} - # THROWAWAY: isolates the two real-subprocess-spawning ProviderRegistry - # suites to find out whether PR #48's opencodeRuntime.ts change (detached - # spawn + process-group kill finalizer) is what's hanging "Test Server 2" - # on every run since. Delete this job once the probe answers that. + # THROWAWAY: isolates each real-subprocess-spawning ProviderRegistry suite + # on its own runner, one file per job, to find out which one (and where in + # it) is hanging "Test Server 2" on every run since PR #48's + # opencodeRuntime.ts change (detached spawn + process-group kill + # finalizer). Delete these jobs once the probe answers that. probe_opencode_hang: - name: Probe OpenCode Hang + name: Probe OpenCode Hang (${{ matrix.file }}) runs-on: ubuntu-latest timeout-minutes: 8 + strategy: + fail-fast: false + matrix: + file: + - ProviderRegistry + - ProviderInstanceRegistryLive steps: - name: Checkout uses: actions/checkout@v6 @@ -141,19 +148,22 @@ jobs: cache: true run-install: true + # Wrapped in a shell-level `timeout` (shorter than the job's own + # timeout-minutes) so the step exits cleanly instead of getting + # hard-killed — a hard kill drops the log entirely, per prior runs. - name: Probe run: | set -o pipefail - vp run --filter helmcode test \ - src/provider/Layers/ProviderRegistry.test.ts \ - src/provider/Layers/ProviderInstanceRegistryLive.test.ts \ + timeout --signal=TERM 300 vp run --filter helmcode test \ + src/provider/Layers/${{ matrix.file }}.test.ts \ -- --reporter=verbose 2>&1 | tee /tmp/probe.log + echo "probe exited with ${PIPESTATUS[0]}" | tee -a /tmp/probe.log - name: Upload probe log if: always() uses: actions/upload-artifact@v4 with: - name: probe-log + name: probe-log-${{ matrix.file }} path: /tmp/probe.log if-no-files-found: ignore From 4041f2a64f8819eb8696972b338a069b35f35ac9 Mon Sep 17 00:00:00 2001 From: Michael Buluma <1452922+buluma@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:32:58 +0300 Subject: [PATCH 4/4] ci: capture process-tree snapshot when the probe hangs Kills vp in the background instead of relying on 'timeout', which only signals vp's direct process and not a detached:true grandchild. If vp doesn't exit on its own within ~300s, snapshots 'ps -ef' before force- killing everything, so the log-upload step always runs and we finally see what's still alive when this hangs. --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffb0dcb0..73bcd5b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,7 +131,6 @@ jobs: matrix: file: - ProviderRegistry - - ProviderInstanceRegistryLive steps: - name: Checkout uses: actions/checkout@v6 @@ -148,16 +147,42 @@ jobs: cache: true run-install: true - # Wrapped in a shell-level `timeout` (shorter than the job's own - # timeout-minutes) so the step exits cleanly instead of getting - # hard-killed — a hard kill drops the log entirely, per prior runs. + # `timeout`'s SIGTERM only reaches vp's direct process, not a + # detached:true grandchild (opencodeRuntime.ts spawns those) — the last + # run sat past 300s with the step still "in_progress", suggesting an + # orphaned child is holding the log pipe open. So: run in the + # background, poll for it dying on our own, and if it doesn't, snapshot + # `ps -ef` (proof of a surviving orphan) and force-kill everything + # before exiting 0 — the step must always complete so the log upload + # step actually runs. - name: Probe run: | - set -o pipefail - timeout --signal=TERM 300 vp run --filter helmcode test \ + set +e + vp run --filter helmcode test \ src/provider/Layers/${{ matrix.file }}.test.ts \ - -- --reporter=verbose 2>&1 | tee /tmp/probe.log - echo "probe exited with ${PIPESTATUS[0]}" | tee -a /tmp/probe.log + -- --reporter=verbose > /tmp/probe.log 2>&1 & + VP_PID=$! + for i in $(seq 1 60); do + sleep 5 + kill -0 "$VP_PID" 2>/dev/null || break + done + if kill -0 "$VP_PID" 2>/dev/null; then + { + echo "=== TIMED OUT after ~300s, vp (pid $VP_PID) still alive ===" + echo "=== process tree snapshot ===" + ps -ef + } | tee -a /tmp/probe.log + pkill -9 -P "$VP_PID" 2>/dev/null + kill -9 "$VP_PID" 2>/dev/null + else + wait "$VP_PID" + echo "probe exited with $?" | tee -a /tmp/probe.log + fi + { + echo "=== final process snapshot before job teardown ===" + ps -ef + } | tee -a /tmp/probe.log + exit 0 - name: Upload probe log if: always()