fix(gmail): retry + defer write-backs instead of dropping them on quota

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {

connectors/gmail/src/gmail-api-retry.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { GmailApi, GmailApiError, isGmailRateLimitError } from "./gmail-api";
+
+/**
+ * Rate-limit / transient retry for {@link GmailApi.call}. Gmail write-backs
+ * (star / read via modifyThread) had NO backoff: a single per-user-per-minute
+ * quota 403 (`rateLimitExceeded`) threw straight through and the change was
+ * dropped (PostHog 019ed581). `call()` now absorbs brief blips in-process and
+ * throws on sustained ones so the caller can defer.
+ */
+describe("GmailApi.call — rate-limit / transient retry", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ });
+
+ it("retries a 429 then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 403 rateLimitExceeded then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: { errors: [{ reason: "rateLimitExceeded" }], message: "Quota exceeded" },
+ }),
+ { status: 403, statusText: "Forbidden" }
+ )
+ )
+ .mockResolvedValueOnce(new Response(null, { status: 204 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").modifyThread("t1", ["STARRED"]);
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toBeUndefined();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("retries a 5xx then succeeds", async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response("oops", { status: 503, statusText: "Service Unavailable" })
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ ok: true }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ await vi.runAllTimersAsync();
+ await expect(p).resolves.toEqual({ ok: true });
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("does NOT retry a 403 that is not a rate-limit (permission error)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "Insufficient Permission" } }), {
+ status: 403,
+ statusText: "Forbidden",
+ })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ name: "GmailApiError",
+ status: 403,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry a 404", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(new Response("missing", { status: 404, statusText: "Not Found" }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/threads/x")).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does NOT retry in-process when Retry-After exceeds the cap (defers instead)", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response("rate", { status: 429, headers: { "Retry-After": "120" } })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(new GmailApi("tok").call("/profile")).rejects.toMatchObject({
+ status: 429,
+ });
+ // A 2-minute wait belongs in the deferred drain, not an in-flight isolate.
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws after exhausting retries on a persistent 429", async () => {
+ vi.useFakeTimers();
+ // Fresh Response per call — a Response body can only be read once.
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(
+ async () =>
+ new Response("rate", { status: 429, statusText: "Too Many Requests" })
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const p = new GmailApi("tok").call("/profile");
+ p.catch(() => {}); // avoid unhandled-rejection noise while timers advance
+ await vi.runAllTimersAsync();
+ await expect(p).rejects.toMatchObject({ status: 429 });
+ expect(fetchMock).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+ });
+});
+
+describe("isGmailRateLimitError", () => {
+ it("is true for HTTP 429", () => {
+ expect(isGmailRateLimitError(new GmailApiError(429, "Too Many Requests", ""))).toBe(true);
+ });
+ it("is true for a 403 rateLimitExceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", '...reason: "rateLimitExceeded"...'))
+ ).toBe(true);
+ });
+ it("is true for a 403 Quota exceeded body", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Quota exceeded for quota metric"))
+ ).toBe(true);
+ });
+ it("is false for a 403 permission error", () => {
+ expect(
+ isGmailRateLimitError(new GmailApiError(403, "Forbidden", "Insufficient Permission"))
+ ).toBe(false);
+ });
+ it("is false for a 404", () => {
+ expect(isGmailRateLimitError(new GmailApiError(404, "Not Found", ""))).toBe(false);
+ });
+ it("is false for a non-GmailApiError that merely mentions the marker", () => {
+ expect(isGmailRateLimitError(new Error("rateLimitExceeded"))).toBe(false);
+ });
+});

connectors/gmail/src/gmail-api.ts

@@ -78,6 +78,39 @@ export class GmailApiError extends Error {
  }
 }
 
+/**
+ * True when a {@link GmailApiError} is a Gmail rate-limit / quota rejection:
+ * an HTTP 429, or a 403 carrying one of Gmail's unambiguous quota markers
+ * (`rateLimitExceeded` / `userRateLimitExceeded` / `Quota exceeded`). These are
+ * expected under load and self-resolve once the per-user-per-minute window
+ * clears, so callers retry/defer them rather than dropping the write-back or
+ * paging error tracking. Gated on the markers — NOT a bare 403 — so genuine
+ * permission failures (`Insufficient Permission`) still surface.
+ */
+export function isGmailRateLimitError(error: unknown): boolean {
+ if (!(error instanceof GmailApiError)) return false;
+ if (error.status === 429) return true;
+ return (
+ error.status === 403 &&
+ /rateLimitExceeded|userRateLimitExceeded|Quota exceeded/i.test(error.message)
+ );
+}
+
+/**
+ * In-process retry budget for {@link GmailApi.call}. Kept small and short so a
+ * brief blip (a momentary 429, a 5xx, a dropped connection) is absorbed inside
+ * the current execution without risking the worker's wall-clock budget. Sustained
+ * rate-limits exceed this and throw, so the caller (deferred write-back drain,
+ * incremental-sync pending list) can reschedule past the quota window.
+ */
+const GMAIL_CALL_MAX_ATTEMPTS = 3;
+const GMAIL_CALL_BACKOFF_MS = [500, 1500];
+/**
+ * Honor a server `Retry-After` only up to this bound; a longer wait belongs in a
+ * scheduled retry, not an in-flight isolate, so we throw and let the caller defer.
+ */
+const GMAIL_RETRY_AFTER_MAX_MS = 3000;
+
 export class GmailApi {
  private baseUrl = "https://gmail.googleapis.com/gmail/v1/users/me";
 
@@ -114,25 +147,96 @@ export class GmailApi {
  "Content-Type": "application/json",
  };
 
- const response = await fetch(url.toString(), {
- method,
- headers,
- body: body ? JSON.stringify(body) : undefined,
- });
+ // Bounded in-process retry for transient failures (rate-limit / 5xx /
+ // dropped connection). A momentary blip is absorbed here; a sustained one
+ // exceeds the budget and throws so the caller can defer past the quota
+ // window (see deferred write-back drain / incremental-sync pending list).
+ let lastError: unknown;
+ for (let attempt = 0; attempt < GMAIL_CALL_MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(url.toString(), {
+ method,
+ headers,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ } catch (networkError) {
+ // fetch() rejects on a dropped/aborted connection — transient.
+ lastError = networkError;
+ if (attempt < GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ await this.sleep(GMAIL_CALL_BACKOFF_MS[attempt] ?? 0);
+ continue;
+ }
+ throw networkError;
+ }
+
+ if (response.ok) {
+ // Some Gmail endpoints — notably users.stop (POST /stop, used by
+ // stopWatch) — return 204 No Content with an EMPTY body. Calling
+ // response.json() on an empty body throws "SyntaxError: Unexpected end
+ // of JSON input"; this escaped through setupWatch()'s unguarded
+ // stopWatch() recovery path and surfaced as an unhandled twist
+ // exception. Read the body as text and only parse it when non-empty.
+ const text = await response.text();
+ return text ? JSON.parse(text) : undefined;
+ }
 
- if (!response.ok) {
  const errorText = await response.text();
- throw new GmailApiError(response.status, response.statusText, errorText);
+ const error = new GmailApiError(
+ response.status,
+ response.statusText,
+ errorText
+ );
+ lastError = error;
+
+ const retryable = isGmailRateLimitError(error) || response.status >= 500;
+ if (!retryable || attempt >= GMAIL_CALL_MAX_ATTEMPTS - 1) {
+ throw error;
+ }
+
+ const delayMs = this.retryDelayMs(response, attempt);
+ // A long server-requested wait belongs in a scheduled retry, not here.
+ if (delayMs === null) throw error;
+ await this.sleep(delayMs);
  }
 
- // Some Gmail endpoints — notably users.stop (POST /stop, used by
- // stopWatch) — return 204 No Content with an EMPTY body. Calling
- // response.json() on an empty body throws "SyntaxError: Unexpected end of
- // JSON input"; this escaped through setupWatch()'s unguarded stopWatch()
- // recovery path and surfaced as an unhandled twist exception. Read the
- // body as text and only parse it when non-empty.
- const text = await response.text();
- return text ? JSON.parse(text) : undefined;
+ // Unreachable in practice (the loop returns or throws), but satisfies the
+ // type checker and surfaces any logic error rather than returning undefined.
+ throw lastError instanceof Error
+ ? lastError
+ : new GmailApiError(0, "Retry loop exhausted", String(lastError));
+ }
+
+ private sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ /**
+ * Backoff for a retryable response. Honors a `Retry-After` header (seconds or
+ * HTTP date) up to {@link GMAIL_RETRY_AFTER_MAX_MS}; returns null when the
+ * server asks for longer than that, signalling the caller to throw and defer
+ * rather than block the isolate. Falls back to a fixed backoff schedule.
+ */
+ private retryDelayMs(response: Response, attempt: number): number | null {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const seconds = Number(retryAfter);
+ let ms: number;
+ if (Number.isFinite(seconds)) {
+ ms = seconds * 1000;
+ } else {
+ const when = Date.parse(retryAfter);
+ ms = Number.isNaN(when) ? NaN : when - Date.now();
+ }
+ if (Number.isFinite(ms)) {
+ if (ms > GMAIL_RETRY_AFTER_MAX_MS) return null;
+ return Math.max(0, ms);
+ }
+ }
+ return (
+ GMAIL_CALL_BACKOFF_MS[attempt] ??
+ GMAIL_CALL_BACKOFF_MS[GMAIL_CALL_BACKOFF_MS.length - 1]
+ );
  }
 
  public async getLabels(): Promise<GmailLabel[]> {