From 14468ba66debdf98bf76e00d136c348d152d28d6 Mon Sep 17 00:00:00 2001
From: Nathan Shan <nathanielcrush51@gmail.com>
Date: Mon, 31 Aug 2026 23:40:37 +0800
Subject: [PATCH] fix(cloud): back off relay connector restarts

Wait before restarting an exited relay connector to prevent rapid crash loops from exhausting the host.

Release the configuration lock during the delay so T3 Connect can still be changed or disabled immediately.

Verify with focused managed endpoint runtime tests and server typecheck.
---
 .../src/cloud/ManagedEndpointRuntime.test.ts  |  9 +++++-
 .../src/cloud/ManagedEndpointRuntime.ts       | 28 +++++++++++++++++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
index b45b5099252a..400f5cb82cde 100644
--- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
@@ -8,6 +8,7 @@ import * as Option from "effect/Option";
 import * as PlatformError from "effect/PlatformError";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
 import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
 import * as RelayClient from "@t3tools/shared/relayClient";
 
@@ -240,7 +241,7 @@ describe("CloudManagedEndpointRuntime", () => {
     }),
   );
 
-  it.effect("supervises the active connector and restarts it after process exit", () =>
+  it.effect("waits before restarting an exited connector", () =>
     Effect.gen(function* () {
       const spawned: Array<number> = [];
       const killed: Array<number> = [];
@@ -275,6 +276,12 @@ describe("CloudManagedEndpointRuntime", () => {
         tunnelId: "tunnel-1",
       });
       yield* Deferred.succeed(firstExit, ChildProcessSpawner.ExitCode(1));
+      yield* Effect.yieldNow;
+
+      expect(spawned).toEqual([400]);
+      yield* TestClock.adjust("4 seconds");
+      expect(spawned).toEqual([400]);
+      yield* TestClock.adjust("1 second");
       yield* Deferred.await(secondSpawned);
 
       expect(started).toMatchObject({ status: "running", pid: 400 });
diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts
index 89c0a23783c0..8b713eb158ed 100644
--- a/apps/server/src/cloud/ManagedEndpointRuntime.ts
+++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts
@@ -99,6 +99,8 @@ const stopConnector = (connector: ActiveConnector | null) =>
       )
     : Effect.void;
 
+const RELAY_CONNECTOR_RESTART_DELAY = "5 seconds";
+
 export const make = Effect.gen(function* () {
   const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
   const relayClient = yield* RelayClient.RelayClient;
@@ -115,14 +117,14 @@ export const make = Effect.gen(function* () {
   const superviseConnector = (connector: ActiveConnector) =>
     Effect.gen(function* () {
       const result = yield* Effect.result(connector.child.exitCode);
-      yield* reconcileSemaphore.withPermits(1)(
+      const restartConfig = yield* reconcileSemaphore.withPermits(1)(
         Effect.gen(function* () {
           const active = yield* Ref.get(activeRef);
           if (
             active?.child.pid !== connector.child.pid ||
             active.configKey !== connector.configKey
           ) {
-            return;
+            return null;
           }
           yield* Ref.set(activeRef, null);
           yield* stopConnector(connector);
@@ -133,7 +135,7 @@ export const make = Effect.gen(function* () {
             desiredConfig.providerKind !== "cloudflare_tunnel" ||
             runtimeConfigKey(desiredConfig) !== connector.configKey
           ) {
-            return;
+            return null;
           }
 
           yield* Effect.logWarning("Relay client exited; restarting", {
@@ -144,6 +146,26 @@ export const make = Effect.gen(function* () {
             tunnelId: connector.config.tunnelId,
             tunnelName: connector.config.tunnelName,
           });
+          return desiredConfig;
+        }),
+      );
+      if (!restartConfig) {
+        return;
+      }
+
+      // Do not hold the reconcile lock while waiting: a user can still change
+      // or disable T3 Connect instead of waiting behind a failed child.
+      yield* Effect.sleep(RELAY_CONNECTOR_RESTART_DELAY);
+      yield* reconcileSemaphore.withPermits(1)(
+        Effect.gen(function* () {
+          const desiredConfig = yield* Ref.get(desiredConfigRef);
+          if (
+            !desiredConfig ||
+            desiredConfig.providerKind !== "cloudflare_tunnel" ||
+            runtimeConfigKey(desiredConfig) !== connector.configKey
+          ) {
+            return;
+          }
           yield* reconcileConfig(desiredConfig);
         }),
       );