fix(run-engine): debounce hot-key lock contention and 5xx feedback loop

.server-changes/debounce-hot-key-lock-contention.md

@@ -0,0 +1,8 @@
+---
+area: webapp
+type: fix
+---
+
+Reduce 5xx feedback loops on hot debounce keys by quantizing `delayUntil`,
+adding an unlocked fast-path skip, and gracefully handling redlock
+contention in `handleDebounce` so the SDK no longer retries into a herd.

apps/webapp/app/env.server.ts

@@ -666,6 +666,21 @@ const EnvironmentSchema = z
  .int()
  .default(60_000 * 60), // 1 hour
 
+ /**
+ * Bucket size in milliseconds used to quantize the newly computed `delayUntil`
+ * in the debounce system. Quantization collapses concurrent triggers on the
+ * same hot debounce key onto the same target time so the unlocked fast-path
+ * skip is effective. Set to 0 to disable. Default: 1000ms (1s).
+ */
+ RUN_ENGINE_DEBOUNCE_QUANTIZE_NEW_DELAY_UNTIL_MS: z.coerce.number().int().min(0).default(1000),
+
+ /**
+ * Whether the unlocked fast-path skip is enabled in the debounce system.
+ * Acts as a kill switch in case the fast-path needs to be disabled in
+ * production without a redeploy. Default: "1" (enabled).
+ */
+ RUN_ENGINE_DEBOUNCE_FAST_PATH_SKIP_ENABLED: z.string().default("1"),
+
  RUN_ENGINE_WORKER_REDIS_HOST: z
  .string()
  .optional()
@@ -837,6 +852,7 @@ const EnvironmentSchema = z
  .default("info"),
  RUN_ENGINE_TREAT_PRODUCTION_EXECUTION_STALLS_AS_OOM: z.string().default("0"),
  RUN_ENGINE_READ_REPLICA_SNAPSHOTS_SINCE_ENABLED: z.string().default("0"),
+ RUN_ENGINE_DEBOUNCE_USE_REPLICA_FOR_FAST_PATH_READ: z.string().default("0"),
 
  /** How long should the presence ttl last */
  DEV_PRESENCE_SSE_TIMEOUT: z.coerce.number().int().default(30_000),

apps/webapp/app/v3/runEngine.server.ts

@@ -214,6 +214,9 @@ function createRunEngine() {
  // Debounce configuration
  debounce: {
  maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS,
+ quantizeNewDelayUntilMs: env.RUN_ENGINE_DEBOUNCE_QUANTIZE_NEW_DELAY_UNTIL_MS,
+ fastPathSkipEnabled: env.RUN_ENGINE_DEBOUNCE_FAST_PATH_SKIP_ENABLED === "1",
+ useReplicaForFastPathRead: env.RUN_ENGINE_DEBOUNCE_USE_REPLICA_FOR_FAST_PATH_READ === "1",
  },
  });
 

internal-packages/run-engine/src/engine/index.ts

@@ -324,6 +324,9 @@ export class RunEngine {
  executionSnapshotSystem: this.executionSnapshotSystem,
  delayedRunSystem: this.delayedRunSystem,
  maxDebounceDurationMs: options.debounce?.maxDebounceDurationMs ?? 60 * 60 * 1000, // Default 1 hour
+ quantizeNewDelayUntilMs: options.debounce?.quantizeNewDelayUntilMs ?? 1000,
+ fastPathSkipEnabled: options.debounce?.fastPathSkipEnabled ?? true,
+ useReplicaForFastPathRead: options.debounce?.useReplicaForFastPathRead ?? false,
  });
 
  this.pendingVersionSystem = new PendingVersionSystem({