feat: OTEL metrics pipeline for task workers

.changeset/fix-coderabbit-review-items.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/sdk": patch
+---
+
+Add OTEL metrics pipeline for task workers. Workers collect process CPU/memory, Node.js runtime metrics (event loop utilization, event loop delay, heap usage), and user-defined custom metrics via `otel.metrics.getMeter()`. Metrics are exported to ClickHouse with 10-second aggregation buckets and 1m/5m rollups, and are queryable through the dashboard query engine with typed attribute columns, `prettyFormat()` for human-readable values, and AI query support.

apps/webapp/app/components/code/QueryResultsChart.tsx

@@ -1,6 +1,9 @@
-import type { OutputColumnMetadata } from "@internal/clickhouse";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/clickhouse";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { BarChart3, LineChart } from "lucide-react";
 import { memo, useMemo } from "react";
+import { createValueFormatter } from "~/utils/columnFormat";
+import { formatCurrencyAccurate } from "~/utils/numberFormatter";
 import type { ChartConfig } from "~/components/primitives/charts/Chart";
 import { Chart } from "~/components/primitives/charts/ChartCompound";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -798,8 +801,24 @@ export const QueryResultsChart = memo(function QueryResultsChart({
  };
  }, [isDateBased, timeGranularity]);
 
- // Create dynamic Y-axis formatter based on data range
- const yAxisFormatter = useMemo(() => createYAxisFormatter(data, series), [data, series]);
+ // Resolve the Y-axis column format for formatting
+ const yAxisFormat = useMemo(() => {
+ if (yAxisColumns.length === 0) return undefined;
+ const col = columns.find((c) => c.name === yAxisColumns[0]);
+ return (col?.format ?? col?.customRenderType) as ColumnFormatType | undefined;
+ }, [yAxisColumns, columns]);
+
+ // Create dynamic Y-axis formatter based on data range and format
+ const yAxisFormatter = useMemo(
+ () => createYAxisFormatter(data, series, yAxisFormat),
+ [data, series, yAxisFormat]
+ );
+
+ // Create value formatter for tooltips and legend based on column format
+ const tooltipValueFormatter = useMemo(
+ () => createValueFormatter(yAxisFormat),
+ [yAxisFormat]
+ );
 
  // Check if the group-by column has a runStatus customRenderType
  const groupByIsRunStatus = useMemo(() => {
@@ -1019,6 +1038,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
  showLegend={showLegend}
  maxLegendItems={fullLegend ? Infinity : 5}
  legendAggregation={config.aggregation}
+ legendValueFormatter={tooltipValueFormatter}
  minHeight="300px"
  fillContainer
  onViewAllLegendItems={onViewAllLegendItems}
@@ -1030,6 +1050,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
  yAxisProps={yAxisProps}
  stackId={stacked ? "stack" : undefined}
  tooltipLabelFormatter={tooltipLabelFormatter}
+ tooltipValueFormatter={tooltipValueFormatter}
  />
  </Chart.Root>
  );
@@ -1046,6 +1067,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
  showLegend={showLegend}
  maxLegendItems={fullLegend ? Infinity : 5}
  legendAggregation={config.aggregation}
+ legendValueFormatter={tooltipValueFormatter}
  minHeight="300px"
  fillContainer
  onViewAllLegendItems={onViewAllLegendItems}
@@ -1057,16 +1079,21 @@ export const QueryResultsChart = memo(function QueryResultsChart({
  yAxisProps={yAxisProps}
  stacked={stacked && sortedSeries.length > 1}
  tooltipLabelFormatter={tooltipLabelFormatter}
+ tooltipValueFormatter={tooltipValueFormatter}
  lineType="linear"
  />
  </Chart.Root>
  );
 });
 
 /**
- * Creates a Y-axis value formatter based on the data range
+ * Creates a Y-axis value formatter based on the data range and optional format hint
  */
-function createYAxisFormatter(data: Record<string, unknown>[], series: string[]) {
+function createYAxisFormatter(
+ data: Record<string, unknown>[],
+ series: string[],
+ format?: ColumnFormatType
+) {
  // Find min and max values across all series
  let minVal = Infinity;
  let maxVal = -Infinity;
@@ -1083,6 +1110,46 @@ function createYAxisFormatter(data: Record<string, unknown>[], series: string[])
 
  const range = maxVal - minVal;
 
+ // Format-aware formatters
+ if (format === "bytes" || format === "decimalBytes") {
+ const divisor = format === "bytes" ? 1024 : 1000;
+ const units =
+ format === "bytes"
+ ? ["B", "KiB", "MiB", "GiB", "TiB"]
+ : ["B", "KB", "MB", "GB", "TB"];
+ return (value: number): string => {
+ if (value === 0) return "0 B";
+ // Use consistent unit for all ticks based on max value
+ const i = Math.min(
+ Math.max(0, Math.floor(Math.log(Math.abs(maxVal || 1)) / Math.log(divisor))),
+ units.length - 1
+ );
+ const scaled = value / Math.pow(divisor, i);
+ return `${scaled.toFixed(scaled < 10 ? 1 : 0)} ${units[i]}`;
+ };
+ }
+
+ if (format === "percent") {
+ return (value: number): string => `${value.toFixed(range < 1 ? 2 : 1)}%`;
+ }
+
+ if (format === "duration") {
+ return (value: number): string => formatDurationMilliseconds(value, { style: "short" });
+ }
+
+ if (format === "durationSeconds") {
+ return (value: number): string =>
+ formatDurationMilliseconds(value * 1000, { style: "short" });
+ }
+
+ if (format === "costInDollars" || format === "cost") {
+ return (value: number): string => {
+ const dollars = format === "cost" ? value / 100 : value;
+ return formatCurrencyAccurate(dollars);
+ };
+ }
+
+ // Default formatter
  return (value: number): string => {
  // Use abbreviations for large numbers
  if (Math.abs(value) >= 1_000_000) {

apps/webapp/app/components/code/TSQLResultsTable.tsx

@@ -35,6 +35,7 @@ import { useCopy } from "~/hooks/useCopy";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { cn } from "~/utils/cn";
+import { formatBytes, formatDecimalBytes, formatQuantity } from "~/utils/columnFormat";
 import { formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter";
 import { v3ProjectPath, v3RunPathFromFriendlyId } from "~/utils/pathBuilder";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -66,9 +67,10 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
  if (value === null) return "NULL";
  if (value === undefined) return "";
 
- // Handle custom render types
- if (column.customRenderType) {
- switch (column.customRenderType) {
+ // Handle format hints (from prettyFormat() or auto-populated from customRenderType)
+ const formatType = column.format ?? column.customRenderType;
+ if (formatType) {
+ switch (formatType) {
  case "duration":
  if (typeof value === "number") {
  return formatDurationMilliseconds(value, { style: "short" });
@@ -95,6 +97,26 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
  return value;
  }
  break;
+ case "bytes":
+ if (typeof value === "number") {
+ return formatBytes(value);
+ }
+ break;
+ case "decimalBytes":
+ if (typeof value === "number") {
+ return formatDecimalBytes(value);
+ }
+ break;
+ case "percent":
+ if (typeof value === "number") {
+ return `${value.toFixed(2)}%`;
+ }
+ break;
+ case "quantity":
+ if (typeof value === "number") {
+ return formatQuantity(value);
+ }
+ break;
  }
  }
 
@@ -222,6 +244,21 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
  if (value === null) return 4; // "NULL"
  if (value === undefined) return 9; // "UNDEFINED"
 
+ // Handle format hint types - estimate their rendered width
+ const fmt = column.format;
+ if (fmt === "bytes" || fmt === "decimalBytes") {
+ // e.g., "1.50 GiB" or "256.00 MB"
+ return 12;
+ }
+ if (fmt === "percent") {
+ // e.g., "45.23%"
+ return 8;
+ }
+ if (fmt === "quantity") {
+ // e.g., "1.50M"
+ return 8;
+ }
+
  // Handle custom render types - estimate their rendered width
  if (column.customRenderType) {
  switch (column.customRenderType) {
@@ -263,6 +300,8 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
  return typeof value === "string" ? Math.min(value.length, 20) : 12;
  case "queue":
  return typeof value === "string" ? Math.min(value.length, 25) : 15;
+ case "deploymentId":
+ return typeof value === "string" ? Math.min(value.length, 25) : 20;
  }
  }
 
@@ -394,6 +433,10 @@ function isRightAlignedColumn(column: OutputColumnMetadata): boolean {
  ) {
  return true;
  }
+ const fmt = column.format;
+ if (fmt === "bytes" || fmt === "decimalBytes" || fmt === "percent" || fmt === "quantity") {
+ return true;
+ }
  return isNumericType(column.type);
 }
 
@@ -476,6 +519,32 @@ function CellValue({
  return <pre className="text-text-dimmed">UNDEFINED</pre>;
  }
 
+ // Check format hint for new format types (from prettyFormat())
+ if (column.format && !column.customRenderType) {
+ switch (column.format) {
+ case "bytes":
+ if (typeof value === "number") {
+ return <span className="tabular-nums">{formatBytes(value)}</span>;
+ }
+ break;
+ case "decimalBytes":
+ if (typeof value === "number") {
+ return <span className="tabular-nums">{formatDecimalBytes(value)}</span>;
+ }
+ break;
+ case "percent":
+ if (typeof value === "number") {
+ return <span className="tabular-nums">{value.toFixed(2)}%</span>;
+ }
+ break;
+ case "quantity":
+ if (typeof value === "number") {
+ return <span className="tabular-nums">{formatQuantity(value)}</span>;
+ }
+ break;
+ }
+ }
+
  // First check customRenderType for special rendering
  if (column.customRenderType) {
  switch (column.customRenderType) {
@@ -577,6 +646,19 @@ function CellValue({
  }
  return <span>{String(value)}</span>;
  }
+ case "deploymentId": {
+ if (typeof value === "string" && value.startsWith("deployment_")) {
+ return (
+ <SimpleTooltip
+ content="Jump to deployment"
+ disableHoverableContent
+ hidden={!hovered}
+ button={<TextLink to={`/deployments/${value}`}>{value}</TextLink>}
+ />
+ );
+ }
+ return <span>{String(value)}</span>;
+ }
  }
  }
 

apps/webapp/app/components/primitives/charts/BigNumberCard.tsx

@@ -1,10 +1,11 @@
-import type { OutputColumnMetadata } from "@internal/tsql";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
 import { Hash } from "lucide-react";
 import { useMemo } from "react";
 import type {
  BigNumberAggregationType,
  BigNumberConfiguration,
 } from "~/components/metrics/QueryWidget";
+import { createValueFormatter } from "~/utils/columnFormat";
 import { AnimatedNumber } from "../AnimatedNumber";
 import { ChartBlankState } from "./ChartBlankState";
 import { Spinner } from "../Spinner";
@@ -130,6 +131,15 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
  return aggregateValues(values, aggregation);
  }, [rows, column, aggregation, sortDirection]);
 
+ // Look up column format for format-aware display
+ const columnValueFormatter = useMemo(() => {
+ const columnMeta = columns.find((c) => c.name === column);
+ const formatType = (columnMeta?.format ?? columnMeta?.customRenderType) as
+ | ColumnFormatType
+ | undefined;
+ return createValueFormatter(formatType);
+ }, [columns, column]);
+
  if (isLoading) {
  return (
  <div className="grid h-full place-items-center [container-type:size]">
@@ -142,14 +152,29 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
  return <ChartBlankState icon={Hash} message="No data to display" />;
  }
 
+ // Use format-aware formatter when available
+ if (columnValueFormatter) {
+ return (
+ <div className="h-full w-full [container-type:size]">
+ <div className="grid h-full w-full place-items-center">
+ <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
+ {prefix && <span>{prefix}</span>}
+ <span>{columnValueFormatter(result)}</span>
+ {suffix && <span className="text-[0.4em] text-text-dimmed">{suffix}</span>}
+ </div>
+ </div>
+ </div>
+ );
+ }
+
  const { displayValue, unitSuffix, decimalPlaces } = abbreviate
  ? abbreviateValue(result)
  : { displayValue: result, unitSuffix: undefined, decimalPlaces: getDecimalPlaces(result) };
 
  return (
  <div className="h-full w-full [container-type:size]">
  <div className="grid h-full w-full place-items-center">
- <div className="flex items-baseline gap-[0.15em] whitespace-nowrap font-normal tabular-nums leading-none text-text-bright text-[clamp(24px,12cqw,96px)]">
+ <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
  {prefix && <span>{prefix}</span>}
  <AnimatedNumber value={displayValue} decimalPlaces={decimalPlaces} />
  {(unitSuffix || suffix) && (

apps/webapp/app/components/primitives/charts/Chart.tsx

@@ -104,6 +104,8 @@ const ChartTooltipContent = React.forwardRef<
  indicator?: "line" | "dot" | "dashed";
  nameKey?: string;
  labelKey?: string;
+ /** Optional formatter for numeric values (e.g. bytes, duration) */
+ valueFormatter?: (value: number) => string;
  }
 >(
  (
@@ -121,6 +123,7 @@ const ChartTooltipContent = React.forwardRef<
  color,
  nameKey,
  labelKey,
+ valueFormatter,
  },
  ref
  ) => {
@@ -221,9 +224,11 @@ const ChartTooltipContent = React.forwardRef<
  {itemConfig?.label || item.name}
  </span>
  </div>
- {item.value && (
+ {item.value != null && (
  <span className="text-foreground font-mono font-medium tabular-nums">
- {item.value.toLocaleString()}
+ {valueFormatter && typeof item.value === "number"
+ ? valueFormatter(item.value)
+ : item.value.toLocaleString()}
  </span>
  )}
  </div>