fix(core): truncate large error stacks and messages to prevent OOM

.changeset/truncate-error-stacks.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Truncate large error stacks and messages to prevent OOM crashes. Stack traces are capped at 50 frames (keeping top 5 + bottom 45 with an omission notice), individual stack lines at 1024 chars, and error messages at 1000 chars. Applied in parseError, sanitizeError, and OTel span recording.

packages/core/src/v3/errors.ts

@@ -154,22 +154,74 @@ export function isCompleteTaskWithOutput(error: unknown): error is CompleteTaskW
  return error instanceof Error && error.name === "CompleteTaskWithOutput";
 }
 
+const MAX_STACK_FRAMES = 50;
+const KEEP_TOP_FRAMES = 5;
+const MAX_STACK_LINE_LENGTH = 1024;
+const MAX_MESSAGE_LENGTH = 1_000;
+
+/** Truncate a stack trace to at most MAX_STACK_FRAMES frames, keeping
+ * the top (closest to throw) and bottom (entry points) frames.
+ * Individual lines (including message lines) are capped at MAX_STACK_LINE_LENGTH
+ * to prevent OOM from huge error messages embedded in the stack. */
+export function truncateStack(stack: string | undefined): string {
+ if (!stack) return "";
+
+ const lines = stack.split("\n");
+
+ // First line(s) before the first frame are the error message
+ const messageLines: string[] = [];
+ const frameLines: string[] = [];
+
+ for (const line of lines) {
+ const safe =
+ line.length > MAX_STACK_LINE_LENGTH
+ ? line.slice(0, MAX_STACK_LINE_LENGTH) + "...[truncated]"
+ : line;
+ if (frameLines.length === 0 && !line.trimStart().startsWith("at ")) {
+ messageLines.push(safe);
+ } else {
+ frameLines.push(safe);
+ }
+ }
+
+ if (frameLines.length <= MAX_STACK_FRAMES) {
+ return [...messageLines, ...frameLines].join("\n");
+ }
+
+ const keepBottom = MAX_STACK_FRAMES - KEEP_TOP_FRAMES;
+ const omitted = frameLines.length - MAX_STACK_FRAMES;
+
+ return [
+ ...messageLines,
+ ...frameLines.slice(0, KEEP_TOP_FRAMES),
+ ` ... ${omitted} frames omitted ...`,
+ ...frameLines.slice(-keepBottom),
+ ].join("\n");
+}
+
+export function truncateMessage(message: string | undefined): string {
+ if (!message) return "";
+ return message.length > MAX_MESSAGE_LENGTH
+ ? message.slice(0, MAX_MESSAGE_LENGTH) + "...[truncated]"
+ : message;
+}
+
 export function parseError(error: unknown): TaskRunError {
  if (isInternalError(error)) {
  return {
  type: "INTERNAL_ERROR",
  code: error.code,
- message: error.message,
- stackTrace: error.stack ?? "",
+ message: truncateMessage(error.message),
+ stackTrace: truncateStack(error.stack),
  };
  }
 
  if (error instanceof Error) {
  return {
  type: "BUILT_IN_ERROR",
  name: error.name,
- message: error.message,
- stackTrace: error.stack ?? "",
+ message: truncateMessage(error.message),
+ stackTrace: truncateStack(error.stack),
  };
  }
 
@@ -248,35 +300,52 @@ export function createJsonErrorObject(error: TaskRunError): SerializedError {
  }
 }
 
-// Removes any null characters from the error message
+// Removes null characters and truncates oversized fields to prevent OOM
 export function sanitizeError(error: TaskRunError): TaskRunError {
  switch (error.type) {
  case "BUILT_IN_ERROR": {
  return {
  type: "BUILT_IN_ERROR",
- message: error.message?.replace(/\0/g, ""),
+ message: truncateMessage(error.message?.replace(/\0/g, "")),
  name: error.name?.replace(/\0/g, ""),
- stackTrace: error.stackTrace?.replace(/\0/g, ""),
+ stackTrace: truncateStack(error.stackTrace?.replace(/\0/g, "")),
  };
  }
  case "STRING_ERROR": {
  return {
  type: "STRING_ERROR",
- raw: error.raw.replace(/\0/g, ""),
+ raw: truncateMessage(error.raw.replace(/\0/g, "")),
  };
  }
  case "CUSTOM_ERROR": {
+ // CUSTOM_ERROR.raw holds JSON.stringify(error) which is later parsed by
+ // JSON.parse in createErrorTaskError. Naive truncation would cut mid-token
+ // and produce invalid JSON — wrap the preview in a valid JSON envelope.
+ const clean = error.raw.replace(/\0/g, "");
+ const safeRaw =
+ clean.length > MAX_MESSAGE_LENGTH
+ ? JSON.stringify({ truncated: true, preview: clean.slice(0, MAX_MESSAGE_LENGTH) })
+ : clean;
  return {
  type: "CUSTOM_ERROR",
- raw: error.raw.replace(/\0/g, ""),
+ raw: safeRaw,
  };
  }
  case "INTERNAL_ERROR": {
+ // message and stackTrace are optional for INTERNAL_ERROR — preserve
+ // `undefined` so the `error.message ?? "Internal error (CODE)"` fallback
+ // in createErrorTaskError still kicks in (empty string is not nullish).
  return {
  type: "INTERNAL_ERROR",
  code: error.code,
- message: error.message?.replace(/\0/g, ""),
- stackTrace: error.stackTrace?.replace(/\0/g, ""),
+ message:
+ error.message != null
+ ? truncateMessage(error.message.replace(/\0/g, ""))
+ : undefined,
+ stackTrace:
+ error.stackTrace != null
+ ? truncateStack(error.stackTrace.replace(/\0/g, ""))
+ : undefined,
  };
  }
  }

packages/core/src/v3/otel/utils.ts

@@ -1,22 +1,47 @@
 import { type Span, SpanStatusCode, context, propagation } from "@opentelemetry/api";
+import { truncateStack, truncateMessage } from "../errors.js";
+
+const MAX_GENERIC_LENGTH = 5_000;
+
+function truncateGeneric(value: string): string {
+ return value.length > MAX_GENERIC_LENGTH
+ ? value.slice(0, MAX_GENERIC_LENGTH) + "...[truncated]"
+ : value;
+}
+
+function serializeFallback(error: unknown): string {
+ // JSON.stringify can throw (circular refs, BigInt) or return undefined
+ // (symbol, undefined, function). Fall back to String() in both cases so we
+ // never mask the original error being recorded.
+ try {
+ const json = JSON.stringify(error);
+ if (json != null) return json;
+ } catch {
+ // fall through
+ }
+ try {
+ return String(error);
+ } catch {
+ return "[unserializable error]";
+ }
+}
 
 export function recordSpanException(span: Span, error: unknown) {
  if (error instanceof Error) {
  span.recordException(sanitizeSpanError(error));
  } else if (typeof error === "string") {
- span.recordException(error.replace(/\0/g, ""));
+ span.recordException(truncateGeneric(error.replace(/\0/g, "")));
  } else {
- span.recordException(JSON.stringify(error).replace(/\0/g, ""));
+ span.recordException(truncateGeneric(serializeFallback(error).replace(/\0/g, "")));
  }
 
  span.setStatus({ code: SpanStatusCode.ERROR });
 }
 
 function sanitizeSpanError(error: Error) {
- // Create a new error object with the same name, message and stack trace
- const sanitizedError = new Error(error.message.replace(/\0/g, ""));
+ const sanitizedError = new Error(truncateMessage(error.message.replace(/\0/g, "")));
  sanitizedError.name = error.name.replace(/\0/g, "");
- sanitizedError.stack = error.stack?.replace(/\0/g, "");
+ sanitizedError.stack = truncateStack(error.stack?.replace(/\0/g, "")) || undefined;
 
  return sanitizedError;
 }

packages/core/src/v3/tracer.ts

@@ -145,11 +145,7 @@ export class TriggerTracer {
  }
 
  if (!spanEnded) {
- if (typeof e === "string" || e instanceof Error) {
- span.recordException(e);
- }
-
- span.setStatus({ code: SpanStatusCode.ERROR });
+ recordSpanException(span, e);
  }
 
  throw e;