feat(web): expose Prometheus metrics for the web process

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
  WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+ // Port the web process serves its Prometheus metrics on.
+ WEB_METRICS_PORT: numberSchema.default(3070),
+
  // Auth
  AUTH_SECRET: z.string(),
  AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
  "posthog-js": "^1.369.0",
  "posthog-node": "^5.24.15",
  "pretty-bytes": "^6.1.1",
+ "prom-client": "^15.1.3",
  "psl": "^1.15.0",
  "react": "19.2.4",
  "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
  await import('./sentry.edge.config');
  }
 
+ if (process.env.NEXT_RUNTIME === 'nodejs') {
+ const { startMetricsServer } = await import('./metricsServer');
+ startMetricsServer();
+ }
+
  if (process.env.NEXT_RUNTIME === 'nodejs') {
  const { initialize } = await import('./initialize');
  await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+ // Guard against a missing port: `listen(undefined)` binds a random one, which
+ // would leave the scrape target silently broken instead of loudly absent.
+ const port = Number(env.WEB_METRICS_PORT);
+ if (!Number.isInteger(port) || port <= 0) {
+ logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+ return undefined;
+ }
+
+ const server = createServer(async (req, res) => {
+ if (req.url !== '/metrics') {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+
+ try {
+ const metrics = await registry.metrics();
+ res.writeHead(200, { 'Content-Type': registry.contentType });
+ res.end(metrics);
+ } catch (error) {
+ logger.error(`Failed to collect metrics: ${error}`);
+ res.writeHead(500);
+ res.end();
+ }
+ });
+
+ // Metrics must never take down the web server, so swallow listen failures
+ // (a port collision, most likely) instead of letting the 'error' event throw.
+ server.on('error', (error) => {
+ logger.error(`Metrics server error: ${error}`);
+ });
+
+ server.listen(port, () => {
+ logger.info(`Web metrics server listening on port ${port}`);
+ });
+
+ return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+ return new Set(
+ output
+ .split('\n')
+ .filter(line => line.length > 0 && !line.startsWith('#'))
+ .map(line => line.split(/[ {]/)[0])
+ );
+};
+
+describe('web promClient', () => {
+ it('exposes the metrics needed to diagnose heap pressure', async () => {
+ const names = metricNames(await registry.metrics());
+
+ expect(names).toContain('nodejs_heap_size_limit_bytes');
+ expect(names).toContain('nodejs_heap_size_used_bytes');
+ expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+ });
+
+ it('registers the gc duration histogram', () => {
+ // Asserted via the registry rather than the rendered output: the histogram
+ // emits no series until a garbage collection has actually been observed.
+ expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+ });
+
+ it('reports a plausible heap size limit', async () => {
+ const output = await registry.metrics();
+ const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+ expect(line).toBeDefined();
+
+ const limit = Number(line!.split(' ')[1]);
+ expect(Number.isFinite(limit)).toBe(true);
+ // Any real V8 heap limit is well above 100MB and well below 100GB.
+ expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+ expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+ });
+
+ it('can be collected repeatedly', async () => {
+ const first = await registry.metrics();
+ const second = await registry.metrics();
+
+ expect(metricNames(first)).toEqual(metricNames(second));
+ });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+ name: 'nodejs_heap_size_limit_bytes',
+ help: 'V8 heap size limit in bytes',
+ collect() {
+ this.set(getHeapStatistics().heap_size_limit);
+ },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+ register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
  posthog-js: "npm:^1.369.0"
  posthog-node: "npm:^5.24.15"
  pretty-bytes: "npm:^6.1.1"
+ prom-client: "npm:^15.1.3"
  psl: "npm:^1.15.0"
  raw-loader: "npm:^4.0.2"
  react: "npm:19.2.4"