diff --git a/.agents/resume b/.agents/resume deleted file mode 100755 index f29d5fe2..00000000 --- a/.agents/resume +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "No persistent services require repair." diff --git a/.agents/setup b/.agents/setup deleted file mode 100755 index 59ad2500..00000000 --- a/.agents/setup +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -mise_bin="$HOME/.local/bin/mise" -export PATH="$HOME/.local/bin:$PATH" - -echo "Installing mise..." -if [[ ! -x "$mise_bin" ]]; then - curl -fsSL https://mise.run | sh -fi - -profile_marker="# Techulus Cloud toolchains managed by mise" -if ! grep -Fqx "$profile_marker" "$HOME/.bash_profile" 2>/dev/null; then - cat >> "$HOME/.bash_profile" <<'EOF' - -# Techulus Cloud toolchains managed by mise -if [[ -x "$HOME/.local/bin/mise" ]]; then - eval "$("$HOME/.local/bin/mise" activate bash)" -fi -EOF -fi - -go_version="$(awk '$1 == "go" { print $2; exit }' "$repo_root/agent/go.mod")" - -echo "Installing repository toolchains..." -"$mise_bin" use --global "go@$go_version" node@24 pnpm@11 -for config in \ - "$repo_root/agent/mise.toml" \ - "$repo_root/web/mise.toml" \ - "$repo_root/docs/mise.toml"; do - "$mise_bin" trust "$config" - ( - cd "$(dirname "$config")" - "$mise_bin" install - ) -done - -echo "Installing web dependencies..." -( - cd "$repo_root/web" - "$mise_bin" exec -- pnpm install --frozen-lockfile -) - -echo "Downloading Go dependencies..." -for module in agent cli deployment/updater; do - ( - cd "$repo_root/$module" - "$mise_bin" exec -- go mod download - ) -done - -echo "Orb setup complete." diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index c9a8d9d9..6f843117 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -43,7 +43,7 @@ jobs: run: go vet ./... - name: Install staticcheck - run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest + run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0 - name: Staticcheck run: ./.bin/staticcheck ./... diff --git a/.github/workflows/cli-ci.yml b/.github/workflows/cli-ci.yml index 3bd81025..1caf8caa 100644 --- a/.github/workflows/cli-ci.yml +++ b/.github/workflows/cli-ci.yml @@ -43,7 +43,7 @@ jobs: run: go vet ./... - name: Install staticcheck - run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest + run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0 - name: Staticcheck run: ./.bin/staticcheck ./... diff --git a/.github/workflows/updater-ci.yml b/.github/workflows/updater-ci.yml index 44d3fe51..6146f048 100644 --- a/.github/workflows/updater-ci.yml +++ b/.github/workflows/updater-ci.yml @@ -45,7 +45,7 @@ jobs: run: go vet ./... - name: Install staticcheck - run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest + run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0 - name: Staticcheck run: ./.bin/staticcheck ./... diff --git a/agent/internal/traefik/reload.go b/agent/internal/traefik/reload.go index fdee5bf0..21a1438a 100644 --- a/agent/internal/traefik/reload.go +++ b/agent/internal/traefik/reload.go @@ -15,6 +15,7 @@ import ( const ( lastReloadSuccessMetric = "traefik_config_last_reload_success" pendingReloadMarkerName = ".routing-reload-pending" + metricsReadyTimeout = 15 * time.Second ) var ( @@ -30,6 +31,22 @@ func LastSuccessfulReload() (time.Time, error) { return readLastSuccessfulReload() } +func waitForMetricsReady(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for { + if _, err := LastSuccessfulReload(); err == nil { + return nil + } else { + lastErr = err + } + if time.Now().After(deadline) { + return fmt.Errorf("traefik metrics did not become ready within %s: %w", timeout, lastErr) + } + time.Sleep(reloadPollInterval) + } +} + func fetchLastSuccessfulReload() (time.Time, error) { response, err := metricsHTTPClient.Get(traefikMetricsURL) if err != nil { diff --git a/agent/internal/traefik/reload_test.go b/agent/internal/traefik/reload_test.go index 5a9bb34e..f3519ac2 100644 --- a/agent/internal/traefik/reload_test.go +++ b/agent/internal/traefik/reload_test.go @@ -20,6 +20,54 @@ traefik_config_last_reload_success 1.725e+09 } } +func TestWaitForMetricsReadyRetriesTemporaryErrors(t *testing.T) { + originalReader := readLastSuccessfulReload + originalPollInterval := reloadPollInterval + t.Cleanup(func() { + readLastSuccessfulReload = originalReader + reloadPollInterval = originalPollInterval + }) + + attempts := 0 + readLastSuccessfulReload = func() (time.Time, error) { + attempts++ + if attempts < 3 { + return time.Time{}, os.ErrNotExist + } + return time.Now(), nil + } + reloadPollInterval = time.Millisecond + + if err := waitForMetricsReady(50 * time.Millisecond); err != nil { + t.Fatal(err) + } + if attempts != 3 { + t.Fatalf("metrics read attempted %d times, want 3", attempts) + } +} + +func TestWaitForMetricsReadyTimesOut(t *testing.T) { + originalReader := readLastSuccessfulReload + originalPollInterval := reloadPollInterval + t.Cleanup(func() { + readLastSuccessfulReload = originalReader + reloadPollInterval = originalPollInterval + }) + + readLastSuccessfulReload = func() (time.Time, error) { + return time.Time{}, os.ErrDeadlineExceeded + } + reloadPollInterval = time.Millisecond + + err := waitForMetricsReady(5 * time.Millisecond) + if err == nil { + t.Fatal("metrics readiness wait unexpectedly succeeded") + } + if !strings.Contains(err.Error(), "traefik metrics did not become ready within 5ms") { + t.Fatalf("unexpected timeout error: %v", err) + } +} + func TestDynamicConfigReloadedRequiresReloadAtOrAfterNewestFile(t *testing.T) { originalDir := dynamicConfigDir originalReader := readLastSuccessfulReload diff --git a/agent/internal/traefik/static.go b/agent/internal/traefik/static.go index 6c9cf2bd..d022a84c 100644 --- a/agent/internal/traefik/static.go +++ b/agent/internal/traefik/static.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "reflect" - "time" "gopkg.in/yaml.v3" ) @@ -16,6 +15,7 @@ const ( metricsEntryPointAddr = "127.0.0.1:9100" ) +// Whole-number buckets must be ints to match yaml.v3's decoded types. var prometheusLatencyBuckets = []interface{}{ 0.005, 0.01, @@ -26,12 +26,12 @@ var prometheusLatencyBuckets = []interface{}{ 0.25, 0.5, 0.75, - 1.0, + 1, 2.5, - 5.0, - 10.0, - 30.0, - 60.0, + 5, + 10, + 30, + 60, } func validateStaticConfig(data []byte) error { @@ -218,6 +218,5 @@ func ReloadTraefik() error { return fmt.Errorf("failed to restart traefik: %w", err) } log.Printf("[traefik] restarted traefik to apply static config changes") - time.Sleep(2 * time.Second) - return nil + return waitForMetricsReady(metricsReadyTimeout) } diff --git a/agent/internal/traefik/static_test.go b/agent/internal/traefik/static_test.go index 7f2b47af..71d99e9e 100644 --- a/agent/internal/traefik/static_test.go +++ b/agent/internal/traefik/static_test.go @@ -1,6 +1,10 @@ package traefik -import "testing" +import ( + "testing" + + "gopkg.in/yaml.v3" +) func TestEnsurePrometheusMetricsConfigAddsPrivateMetricsEndpoint(t *testing.T) { config := map[string]interface{}{ @@ -44,7 +48,17 @@ func TestEnsurePrometheusMetricsConfigIsStable(t *testing.T) { if !ensurePrometheusMetricsConfig(config) { t.Fatal("expected first call to modify config") } - if ensurePrometheusMetricsConfig(config) { + + data, err := yaml.Marshal(config) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + var roundTripped map[string]interface{} + if err := yaml.Unmarshal(data, &roundTripped); err != nil { + t.Fatalf("failed to unmarshal config: %v", err) + } + + if ensurePrometheusMetricsConfig(roundTripped) { t.Fatal("expected second call to be stable") } } diff --git a/web/actions/compose.ts b/web/actions/compose.ts index ed7afa64..4e672cd3 100644 --- a/web/actions/compose.ts +++ b/web/actions/compose.ts @@ -5,6 +5,7 @@ import { db } from "@/db"; import { services } from "@/db/schema"; import { requireDeveloperRole } from "@/lib/auth"; import { parseComposeYaml } from "@/lib/compose-parser"; +import { reportServerError } from "@/lib/server-errors"; import { addServiceVolume, createService, @@ -144,6 +145,9 @@ export async function importCompose( volume.containerPath, ); } catch (e) { + reportServerError(e, "compose-import.volume.add", { + tags: { serviceId: result.id }, + }); warnings.push({ service: finalName, field: "volumes", @@ -182,9 +186,16 @@ export async function importCompose( for (const serviceId of createdServiceIds) { try { await db.delete(services).where(eq(services.id, serviceId)); - } catch {} + } catch (cleanupError) { + reportServerError(cleanupError, "compose-import.cleanup", { + tags: { serviceId }, + }); + } } + reportServerError(error, "compose-import.create", { + tags: { projectId, environmentId }, + }); return { success: false, created: [], diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 7c9bb68f..10b44b35 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -54,6 +54,7 @@ import { cleanupRegistryArtifactsForService, prepareRegistryArtifactCleanup, } from "@/lib/registry-retention"; +import { reportServerError } from "@/lib/server-errors"; import { deletePreviewService, deletePreviewsForBaseService, @@ -1748,6 +1749,9 @@ export async function abortRollout(serviceId: string) { try { await inngest.send(inngestEvents.rolloutCancelled.create({ rolloutId })); } catch (error) { + reportServerError(error, "rollout.cancellation.dispatch", { + tags: { rolloutId, serviceId }, + }); console.error( `[rollout:${rolloutId}] failed to send cancellation:`, error, diff --git a/web/app/api/builds/[buildId]/logs/route.ts b/web/app/api/builds/[buildId]/logs/route.ts index eb68ca32..8133b216 100644 --- a/web/app/api/builds/[buildId]/logs/route.ts +++ b/web/app/api/builds/[buildId]/logs/route.ts @@ -1,5 +1,6 @@ import { type NextRequest, NextResponse } from "next/server"; import { invalidLogQueryResponse, normalizeLogSearch } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, queryLogsByBuild } from "@/lib/victoria-logs"; export async function GET( @@ -28,6 +29,7 @@ export async function GET( return NextResponse.json({ logs }); } catch (error) { + reportServerError(error, "logs.build.query", { tags: { buildId } }); console.error("Failed to fetch build logs:", error); return NextResponse.json( { message: "Failed to query build logs" }, diff --git a/web/app/api/deployments/[id]/logs/route.ts b/web/app/api/deployments/[id]/logs/route.ts index f7c7c328..db4eda2a 100644 --- a/web/app/api/deployments/[id]/logs/route.ts +++ b/web/app/api/deployments/[id]/logs/route.ts @@ -5,6 +5,7 @@ import { normalizeLogCursor, parseLogLimit, } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, queryLogsByDeployment } from "@/lib/victoria-logs"; export async function GET( @@ -50,6 +51,9 @@ export async function GET( hasMore: result.hasMore, }); } catch (error) { + reportServerError(error, "logs.deployment.query", { + tags: { deploymentId }, + }); console.error("[logs:deployment] failed to query logs:", error); return Response.json( { message: "Failed to query deployment logs" }, diff --git a/web/app/api/github/repos/route.ts b/web/app/api/github/repos/route.ts index 6c3bff00..c64e85b7 100644 --- a/web/app/api/github/repos/route.ts +++ b/web/app/api/github/repos/route.ts @@ -4,6 +4,7 @@ import { db } from "@/db"; import { githubInstallations } from "@/db/schema"; import { eq } from "drizzle-orm"; import { getInstallationRepositories } from "@/lib/github"; +import { reportServerError } from "@/lib/server-errors"; export async function GET() { const session = await auth.api.getSession({ @@ -48,6 +49,9 @@ export async function GET() { }); } } catch (error) { + reportServerError(error, "github.repositories.list", { + tags: { installationId: installation.installationId }, + }); console.error( `[github:repos] failed to fetch repos for installation ${installation.installationId}:`, error, diff --git a/web/app/api/github/setup/route.ts b/web/app/api/github/setup/route.ts index fa3dfee5..6ef77f26 100644 --- a/web/app/api/github/setup/route.ts +++ b/web/app/api/github/setup/route.ts @@ -5,6 +5,7 @@ import { type NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; import { githubInstallations } from "@/db/schema"; import { requireRequestDeveloperRole } from "@/lib/api-auth"; +import { reportServerError } from "@/lib/server-errors"; async function getInstallationDetails(installationId: number): Promise<{ account: { login: string; type: "User" | "Organization" }; @@ -40,6 +41,13 @@ async function getInstallationDetails(installationId: number): Promise<{ ); if (!response.ok) { + reportServerError( + new Error( + `GitHub installation lookup failed with status ${response.status}`, + ), + "github.installation.get", + { tags: { installationId } }, + ); console.error( `[github:setup] failed to get installation ${installationId}:`, await response.text(), diff --git a/web/app/api/inngest/route.ts b/web/app/api/inngest/route.ts index 977106bd..b77b7076 100644 --- a/web/app/api/inngest/route.ts +++ b/web/app/api/inngest/route.ts @@ -25,6 +25,7 @@ import { rolloutWorkflow, scheduledBackupsCheck, scheduledDeploymentsCheck, + sentryFailureWorkflow, serviceDeletionWorkflow, serviceCommandRetention, serviceCronDispatcher, @@ -52,6 +53,7 @@ export const { GET, POST, PUT } = serve({ agentUpgradeTimeoutCheck, registryArtifactRetention, migrationWorkflow, + sentryFailureWorkflow, backupWorkflow, restoreWorkflow, onRestoreFailed, diff --git a/web/app/api/rollouts/[rolloutId]/logs/route.ts b/web/app/api/rollouts/[rolloutId]/logs/route.ts index 664c12bf..067607ab 100644 --- a/web/app/api/rollouts/[rolloutId]/logs/route.ts +++ b/web/app/api/rollouts/[rolloutId]/logs/route.ts @@ -1,5 +1,6 @@ import { type NextRequest, NextResponse } from "next/server"; import { invalidLogQueryResponse, normalizeLogSearch } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, queryLogsByRollout } from "@/lib/victoria-logs"; export async function GET( @@ -29,6 +30,7 @@ export async function GET( return NextResponse.json({ logs }); } catch (error) { + reportServerError(error, "logs.rollout.query", { tags: { rolloutId } }); console.error("Failed to fetch rollout logs:", error); return NextResponse.json( { message: "Failed to query rollout logs" }, diff --git a/web/app/api/servers/[id]/logs/route.ts b/web/app/api/servers/[id]/logs/route.ts index a3f59d16..14d687e0 100644 --- a/web/app/api/servers/[id]/logs/route.ts +++ b/web/app/api/servers/[id]/logs/route.ts @@ -1,6 +1,7 @@ import { headers } from "next/headers"; import { auth } from "@/lib/auth"; import { invalidLogQueryResponse, parseLogListParams } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, queryLogsByServer } from "@/lib/victoria-logs"; export async function GET( @@ -46,6 +47,7 @@ export async function GET( hasMore: result.hasMore, }); } catch (error) { + reportServerError(error, "logs.server.query", { tags: { serverId } }); console.error("[logs:server] failed to query logs:", error); return Response.json( { message: "Failed to query server logs" }, diff --git a/web/app/api/servers/[id]/metrics/route.ts b/web/app/api/servers/[id]/metrics/route.ts index 2160acbb..38e1a2b2 100644 --- a/web/app/api/servers/[id]/metrics/route.ts +++ b/web/app/api/servers/[id]/metrics/route.ts @@ -1,6 +1,7 @@ import { headers } from "next/headers"; import { getServerDetails } from "@/db/queries"; import { auth } from "@/lib/auth"; +import { reportServerError } from "@/lib/server-errors"; import { emptyHistory, getMetricWindow, @@ -61,6 +62,7 @@ export async function GET( range, }); } catch (error) { + reportServerError(error, "metrics.server.query", { tags: { serverId } }); console.error("[metrics:server] failed to query metrics:", error); return Response.json({ current: null, diff --git a/web/app/api/services/[id]/backups/route.ts b/web/app/api/services/[id]/backups/route.ts index 6d265aa0..83252e1c 100644 --- a/web/app/api/services/[id]/backups/route.ts +++ b/web/app/api/services/[id]/backups/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { desc, eq } from "drizzle-orm"; import { db } from "@/db"; import { volumeBackups, servers } from "@/db/schema"; +import { reportServerError } from "@/lib/server-errors"; export async function GET( request: NextRequest, @@ -28,6 +29,8 @@ export async function GET( return NextResponse.json({ backups }); } catch (error) { + const { id: serviceId } = await params; + reportServerError(error, "backups.list", { tags: { serviceId } }); console.error("[api:backups] failed to fetch backups:", error); return NextResponse.json( { error: "Failed to fetch backups" }, diff --git a/web/app/api/services/[id]/github/commits/route.ts b/web/app/api/services/[id]/github/commits/route.ts index efc39087..2951d4ca 100644 --- a/web/app/api/services/[id]/github/commits/route.ts +++ b/web/app/api/services/[id]/github/commits/route.ts @@ -3,6 +3,7 @@ import { db } from "@/db"; import { githubRepos, services } from "@/db/schema"; import { requireRequestDeveloperRole } from "@/lib/api-auth"; import { listGitHubCommits } from "@/lib/github"; +import { reportServerError } from "@/lib/server-errors"; export async function GET( request: Request, @@ -41,6 +42,12 @@ export async function GET( ); return Response.json({ branch, commits }); } catch (error) { + reportServerError(error, "github.commits.list", { + tags: { + installationId: result.githubRepo.installationId, + serviceId, + }, + }); return Response.json( { message: diff --git a/web/app/api/services/[id]/logs/route.ts b/web/app/api/services/[id]/logs/route.ts index 99a620d7..65c609ca 100644 --- a/web/app/api/services/[id]/logs/route.ts +++ b/web/app/api/services/[id]/logs/route.ts @@ -1,6 +1,7 @@ import { headers } from "next/headers"; import { auth } from "@/lib/auth"; import { invalidLogQueryResponse, parseLogListParams } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, type LogType, @@ -79,6 +80,7 @@ export async function GET( hasMore: result.hasMore, }); } catch (error) { + reportServerError(error, "logs.service.query", { tags: { serviceId } }); console.error("[logs:service] failed to query logs:", error); return Response.json( { message: "Failed to query service logs" }, diff --git a/web/app/api/services/[id]/metrics/route.ts b/web/app/api/services/[id]/metrics/route.ts index 1d8cc791..0fcecd6e 100644 --- a/web/app/api/services/[id]/metrics/route.ts +++ b/web/app/api/services/[id]/metrics/route.ts @@ -1,6 +1,7 @@ import { headers } from "next/headers"; import { getService } from "@/db/queries"; import { auth } from "@/lib/auth"; +import { reportServerError } from "@/lib/server-errors"; import { createEmptyServiceMetrics, isMetricsEnabled, @@ -45,6 +46,9 @@ export async function GET( try { return Response.json(await queryServiceMetrics({ serviceId, range })); } catch (error) { + reportServerError(error, "metrics.service.query", { + tags: { serviceId }, + }); console.error("[metrics:service] failed to query service metrics:", error); return Response.json( { message: "Service metrics unavailable" }, diff --git a/web/app/api/services/[id]/requests/route.ts b/web/app/api/services/[id]/requests/route.ts index bd03415c..c732506f 100644 --- a/web/app/api/services/[id]/requests/route.ts +++ b/web/app/api/services/[id]/requests/route.ts @@ -1,6 +1,7 @@ import { headers } from "next/headers"; import { auth } from "@/lib/auth"; import { invalidLogQueryResponse, parseLogListParams } from "@/lib/log-query"; +import { reportServerError } from "@/lib/server-errors"; import { isLoggingEnabled, queryLogsByService } from "@/lib/victoria-logs"; export async function GET( @@ -50,6 +51,7 @@ export async function GET( hasMore: result.hasMore, }); } catch (error) { + reportServerError(error, "logs.requests.query", { tags: { serviceId } }); console.error("[logs:requests] failed to query HTTP logs:", error); return Response.json( { message: "Failed to query request logs" }, diff --git a/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts b/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts index a42d2863..18404dc3 100644 --- a/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts +++ b/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts @@ -4,6 +4,7 @@ import { secrets } from "@/db/schema"; import { requireRequestDeveloperRole } from "@/lib/api-auth"; import { decryptSecret } from "@/lib/crypto"; import { EncryptionKeyUnavailableError } from "@/lib/kms"; +import { reportServerError } from "@/lib/server-errors"; export async function POST( request: Request, @@ -40,6 +41,9 @@ export async function POST( }, }); } catch (error) { + reportServerError(error, "secrets.reveal", { + tags: { serviceId, secretId }, + }); if (error instanceof EncryptionKeyUnavailableError) { console.error("Secret encryption key unavailable:", error); return Response.json( diff --git a/web/app/api/v1/agent/backup/failed/route.ts b/web/app/api/v1/agent/backup/failed/route.ts index 48eec3e8..dc66278d 100644 --- a/web/app/api/v1/agent/backup/failed/route.ts +++ b/web/app/api/v1/agent/backup/failed/route.ts @@ -6,6 +6,7 @@ import { volumeBackups } from "@/db/schema"; import { verifyAgentRequest } from "@/lib/agent-auth"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { reportOperationFailure } from "@/lib/server-errors"; export async function POST(request: NextRequest) { const body = await request.text(); @@ -49,6 +50,11 @@ export async function POST(request: NextRequest) { return NextResponse.json({ ok: true }); } + reportOperationFailure("backup.failed", { + occurrenceId: backupId, + reason: "agent_reported_failure", + tags: { backupId, serviceId: backup.serviceId, serverId }, + }); revalidatePath("/dashboard/projects"); await inngest.send( diff --git a/web/app/api/v1/agent/builds/[id]/route.ts b/web/app/api/v1/agent/builds/[id]/route.ts index f54080fd..051675ce 100644 --- a/web/app/api/v1/agent/builds/[id]/route.ts +++ b/web/app/api/v1/agent/builds/[id]/route.ts @@ -8,6 +8,7 @@ import { cloneUrlForRevisionSource } from "@/lib/build-revision-source"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { DEFAULT_BUILD_TIMEOUT_MINUTES, SETTING_KEYS, @@ -68,6 +69,16 @@ export async function POST( { status: 409 }, ); } + reportOperationFailure("build.failed", { + occurrenceId: buildId, + reason: "claim_failed", + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId, + }, + }); await inngest.send( inngestEvents.buildCompleted.create( { @@ -115,6 +126,14 @@ export async function POST( try { specification = parseServiceRevisionSpec(revision.specification); } catch (error) { + reportServerError(error, "agent.build.claim.parse-revision", { + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId, + }, + }); console.error("[build:get] invalid service revision:", error); return failClaim("Invalid build service revision"); } @@ -130,6 +149,14 @@ export async function POST( try { cloneUrl = await cloneUrlForRevisionSource(specification.source); } catch (error) { + reportServerError(error, "agent.build.claim.github-token", { + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId, + }, + }); console.error("[build:get] failed to get installation token:", error); return failClaim("Failed to get GitHub installation token"); } diff --git a/web/app/api/v1/agent/builds/[id]/status/route.ts b/web/app/api/v1/agent/builds/[id]/status/route.ts index d6e48b21..f43afea4 100644 --- a/web/app/api/v1/agent/builds/[id]/status/route.ts +++ b/web/app/api/v1/agent/builds/[id]/status/route.ts @@ -16,6 +16,7 @@ import { inngestEvents } from "@/lib/inngest/events"; import { notify } from "@/lib/notifications"; import { updatePreviewGitHubStatus } from "@/lib/preview-deployments"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { enqueueWork } from "@/lib/work-queue"; type StatusUpdate = { @@ -143,6 +144,14 @@ export async function POST( try { specification = parseServiceRevisionSpec(revision.specification); } catch (error) { + reportServerError(error, "agent.build.status.parse-revision", { + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId: auth.serverId, + }, + }); console.error("[build:status] invalid service revision:", error); return NextResponse.json( { error: "Invalid build service revision" }, @@ -324,6 +333,14 @@ export async function POST( } } } catch (error) { + reportServerError(error, "agent.build.status.github-deployment", { + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId: auth.serverId, + }, + }); console.error( "[build:status] failed to update GitHub deployment:", error, @@ -333,6 +350,16 @@ export async function POST( if (update.status === "failed") { if (!replayingTerminalUpdate) { + reportOperationFailure("build.failed", { + occurrenceId: buildId, + reason: "agent_reported_failure", + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + serverId: auth.serverId, + }, + }); notify({ kind: "build.failed", occurrenceId: buildId, @@ -340,6 +367,13 @@ export async function POST( buildId, error: update.error, }).catch((error) => { + reportServerError(error, "agent.build.status.notification", { + tags: { + buildId, + serviceId: build.serviceId, + revisionId: build.serviceRevisionId, + }, + }); console.error( "[build:status] failed to enqueue build failure notification:", error, diff --git a/web/app/api/v1/agent/register/route.ts b/web/app/api/v1/agent/register/route.ts index 4a095b87..5d5de582 100644 --- a/web/app/api/v1/agent/register/route.ts +++ b/web/app/api/v1/agent/register/route.ts @@ -5,6 +5,7 @@ import { servers } from "@/db/schema"; import { HOUR_IN_MILLISECONDS, subtractMilliseconds } from "@/lib/date"; import { EncryptionKeyUnavailableError, resolveEncryptionKey } from "@/lib/kms"; import { agentRegisterSchema } from "@/lib/schemas"; +import { reportServerError } from "@/lib/server-errors"; import { formatZodErrors } from "@/lib/utils"; import { assignSubnet, @@ -114,6 +115,10 @@ export async function POST(request: NextRequest) { metricsEndpoint: process.env.VICTORIA_METRICS_URL ?? null, }); } catch (error) { + if (error instanceof SyntaxError) { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } + reportServerError(error, "agent.register"); console.error("Agent registration error:", error); if (error instanceof EncryptionKeyUnavailableError) { return NextResponse.json( diff --git a/web/app/api/v1/agent/registries/route.ts b/web/app/api/v1/agent/registries/route.ts index dab33ffd..d2c95fa1 100644 --- a/web/app/api/v1/agent/registries/route.ts +++ b/web/app/api/v1/agent/registries/route.ts @@ -1,6 +1,7 @@ import { type NextRequest, NextResponse } from "next/server"; import { verifyAgentRequest } from "@/lib/agent-auth"; import { getRegistryBundle } from "@/lib/registry-credentials"; +import { reportServerError } from "@/lib/server-errors"; const PRIVATE_HEADERS = { "Cache-Control": "private, no-store", @@ -19,6 +20,9 @@ export async function GET(request: NextRequest) { headers: PRIVATE_HEADERS, }); } catch (error) { + reportServerError(error, "agent.registries.bundle", { + tags: { serverId: auth.serverId }, + }); console.error("Registry bundle error:", error); return NextResponse.json( { error: "Registry credentials unavailable" }, diff --git a/web/app/api/v1/agent/work/wait/route.ts b/web/app/api/v1/agent/work/wait/route.ts index 69735941..dd607cfe 100644 --- a/web/app/api/v1/agent/work/wait/route.ts +++ b/web/app/api/v1/agent/work/wait/route.ts @@ -1,5 +1,6 @@ import { type NextRequest, NextResponse } from "next/server"; import { verifyAgentRequest } from "@/lib/agent-auth"; +import { reportServerError } from "@/lib/server-errors"; import { hasClaimableWork } from "@/lib/work-queue"; import { subscribeToWorkNotifications, @@ -18,12 +19,18 @@ export async function GET(request: NextRequest) { try { subscription = await subscribeToWorkNotifications(auth.serverId); } catch (error) { + reportServerError(error, "agent.work.subscribe", { + tags: { serverId: auth.serverId }, + }); console.error("[work-queue] failed to subscribe for notifications:", error); try { if (await hasClaimableWork(auth.serverId)) { return NextResponse.json({ workAvailable: true }); } } catch (queryError) { + reportServerError(queryError, "agent.work.query", { + tags: { serverId: auth.serverId }, + }); console.error("[work-queue] failed to check for work:", queryError); } return workWaitUnavailable(); @@ -50,6 +57,9 @@ export async function GET(request: NextRequest) { } return NextResponse.json({ workAvailable: false }); } catch (error) { + reportServerError(error, "agent.work.wait", { + tags: { serverId: auth.serverId }, + }); console.error("[work-queue] failed while waiting for work:", error); return workWaitUnavailable(); } finally { diff --git a/web/app/api/v1/api-keys/route.ts b/web/app/api/v1/api-keys/route.ts index 3d23b0a8..f981e33b 100644 --- a/web/app/api/v1/api-keys/route.ts +++ b/web/app/api/v1/api-keys/route.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { requireRequestRole } from "@/lib/api-auth"; import { auth as betterAuth } from "@/lib/auth"; import { apiError, badRequest } from "@/lib/public-api"; +import { reportServerError } from "@/lib/server-errors"; const schema = z.strictObject({ name: z.string().trim().min(1).max(32), @@ -33,6 +34,7 @@ export async function POST(request: Request) { { status: 201 }, ); } catch (error) { + reportServerError(error, "public-api.api-key.create"); console.error("[public-api] API key creation failed", error); return apiError("Failed to create API key", "API_KEY_CREATE_FAILED", 500); } diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts index 310cf593..e15815c1 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts @@ -9,6 +9,7 @@ import { resolvePersistedSource, } from "@/lib/public-api"; import { namedPage, nextNamedCursor } from "@/lib/public-api-pagination"; +import { reportServerError } from "@/lib/server-errors"; export async function GET( request: Request, @@ -79,6 +80,9 @@ export async function GET( nextCursor: nextNamedCursor(rows, page.limit), }); } catch (error) { + reportServerError(error, "public-api.services.list", { + tags: { projectId, environmentId }, + }); console.error("[public-api] list services failed", error); return apiError("Internal server error", "INTERNAL_ERROR", 500); } diff --git a/web/app/api/v1/projects/[projectId]/environments/route.ts b/web/app/api/v1/projects/[projectId]/environments/route.ts index ee66b75f..2b000cd3 100644 --- a/web/app/api/v1/projects/[projectId]/environments/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/route.ts @@ -4,6 +4,7 @@ import { environments, projects } from "@/db/schema"; import { requireApiKeyRole } from "@/lib/api-auth"; import { apiError, badRequest, notFound } from "@/lib/public-api"; import { namedPage, nextNamedCursor } from "@/lib/public-api-pagination"; +import { reportServerError } from "@/lib/server-errors"; export async function GET( request: Request, @@ -58,6 +59,9 @@ export async function GET( nextCursor: nextNamedCursor(items, page.limit), }); } catch (error) { + reportServerError(error, "public-api.environments.list", { + tags: { projectId }, + }); console.error("[public-api] list environments failed", error); return apiError("Internal server error", "INTERNAL_ERROR", 500); } diff --git a/web/app/api/v1/projects/route.ts b/web/app/api/v1/projects/route.ts index 61a3fcda..8e29afc5 100644 --- a/web/app/api/v1/projects/route.ts +++ b/web/app/api/v1/projects/route.ts @@ -4,6 +4,7 @@ import { projects } from "@/db/schema"; import { requireApiKeyRole } from "@/lib/api-auth"; import { apiError, badRequest } from "@/lib/public-api"; import { namedPage, nextNamedCursor } from "@/lib/public-api-pagination"; +import { reportServerError } from "@/lib/server-errors"; export async function GET(request: Request) { const auth = await requireApiKeyRole(request, [ @@ -45,6 +46,7 @@ export async function GET(request: Request) { nextCursor: nextNamedCursor(rows, page.limit), }); } catch (error) { + reportServerError(error, "public-api.projects.list"); console.error("[public-api] list projects failed", error); return apiError("Internal server error", "INTERNAL_ERROR", 500); } diff --git a/web/app/api/webhooks/github/route.ts b/web/app/api/webhooks/github/route.ts index 8e4efedb..13f47159 100644 --- a/web/app/api/webhooks/github/route.ts +++ b/web/app/api/webhooks/github/route.ts @@ -18,6 +18,7 @@ import { import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { deletePreviewsForGitHubInstallation } from "@/lib/preview-lifecycle"; +import { reportServerError } from "@/lib/server-errors"; import { pullRequestMergeRef } from "@/lib/service-revision-spec"; import { triggerResolvedBuildInternal } from "@/lib/trigger-build"; @@ -239,6 +240,12 @@ async function handlePushEvent(payload: PushPayload) { { description: "Build queued", environmentUrl: serviceUrl }, ); } catch (error) { + reportServerError(error, "github.webhook.deployment.create", { + tags: { + installationId: githubRepo.installationId, + serviceId: service.id, + }, + }); console.error( `[webhook:push] failed to create GitHub deployment for service ${service.id}:`, error, @@ -263,6 +270,12 @@ async function handlePushEvent(payload: PushPayload) { results.push({ serviceId: service.id, status: "queued" }); } catch (error) { + reportServerError(error, "github.webhook.build.dispatch", { + tags: { + installationId: githubRepo.installationId, + serviceId: service.id, + }, + }); console.error( `[webhook:push] failed to queue build for service ${service.id}:`, error, @@ -396,6 +409,9 @@ async function handlePullRequestEvent( try { await inngest.send(events); } catch (error) { + reportServerError(error, "github.webhook.preview.dispatch", { + extra: { eventCount: events.length }, + }); console.error("Failed to dispatch preview deployment events:", error); return NextResponse.json( { ok: false, error: "Failed to queue preview deployment work" }, @@ -420,7 +436,12 @@ export async function POST(request: NextRequest) { } const event = request.headers.get("x-github-event"); - const payload = JSON.parse(body); + let payload: unknown; + try { + payload = JSON.parse(body); + } catch { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } console.log(`[webhook:github] received event: ${event}`); diff --git a/web/lib/acme-manager.ts b/web/lib/acme-manager.ts index c4d0314f..7e3ff6b8 100644 --- a/web/lib/acme-manager.ts +++ b/web/lib/acme-manager.ts @@ -10,6 +10,7 @@ import { isExpired, MINUTE_IN_MILLISECONDS, } from "@/lib/date"; +import { reportServerError } from "@/lib/server-errors"; import { SETTING_KEYS } from "@/lib/settings-keys"; const ACME_ACCOUNT_KEY_SETTING = "acme_account_key"; @@ -220,6 +221,9 @@ export async function renewExpiringCertificates(): Promise { await issueCertificate(cert.domain); console.log(`[acme] renewed certificate for ${cert.domain}`); } catch (error) { + reportServerError(error, "acme.certificate.renew", { + tags: { domain: cert.domain }, + }); console.error(`[acme] failed to renew ${cert.domain}:`, error); } } diff --git a/web/lib/api-auth.ts b/web/lib/api-auth.ts index 6346feeb..5222e7e0 100644 --- a/web/lib/api-auth.ts +++ b/web/lib/api-auth.ts @@ -8,6 +8,7 @@ import { getUserRole, hasAnyRole, } from "@/lib/members"; +import { reportServerError } from "@/lib/server-errors"; type AuthenticatedIdentity = { user: { id: string; name: string; email: string }; @@ -59,6 +60,7 @@ function unauthorized() { } function authProviderError(error: unknown) { + reportServerError(error, "authentication.provider"); console.error("[public-api] authentication failed", error); return { ok: false as const, @@ -84,6 +86,9 @@ export async function requireRequestSession( } catch (error) { const response = getAuthErrorResponse(error); if (response) { + if (response.status >= 500) { + reportServerError(error, "authentication.session"); + } return { ok: false as const, response, @@ -155,6 +160,9 @@ export async function requireApiKeySession(request: Request) { } catch (error) { const response = getAuthErrorResponse(error); if (response) { + if (response.status >= 500) { + reportServerError(error, "authentication.api-key"); + } return response.status >= 400 && response.status < 500 && response.status !== 429 @@ -185,6 +193,9 @@ async function requireSessionRole( }; } + reportServerError(error, "authorization.role-lookup", { + tags: { userId: sessionResult.session.user.id }, + }); console.error("[public-api] authorization lookup failed", error); return { ok: false as const, diff --git a/web/lib/backup-scheduler.ts b/web/lib/backup-scheduler.ts index 1b92aa5f..7f95d65a 100644 --- a/web/lib/backup-scheduler.ts +++ b/web/lib/backup-scheduler.ts @@ -9,6 +9,7 @@ import { differenceInElapsedHours, subtractUtcDays, } from "@/lib/date"; +import { reportServerError } from "@/lib/server-errors"; import { DEFAULT_BACKUP_RETENTION_DAYS } from "@/lib/settings-keys"; function shouldRunSchedule( @@ -114,6 +115,9 @@ export async function runScheduledBackups() { ); } } catch (err) { + reportServerError(err, "backup.schedule", { + tags: { serviceId: service.id }, + }); console.error( `[backup-scheduler] error scheduling backup for ${service.name}:`, err, diff --git a/web/lib/backups/delete-backup.ts b/web/lib/backups/delete-backup.ts index 34f36ee2..c853903f 100644 --- a/web/lib/backups/delete-backup.ts +++ b/web/lib/backups/delete-backup.ts @@ -3,6 +3,7 @@ import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { volumeBackups } from "@/db/schema"; import { deleteFromS3 } from "@/lib/s3"; +import { reportServerError } from "@/lib/server-errors"; /** * Deletes a backup record and its S3 object. @@ -27,6 +28,9 @@ export async function deleteBackupInternal(backupId: string) { try { await deleteFromS3(storageConfig.bucket, backup.storagePath); } catch (err) { + reportServerError(err, "backup.storage.delete", { + tags: { backupId }, + }); console.error("[deleteBackup] failed to delete from S3:", { backupId, storagePath: backup.storagePath, diff --git a/web/lib/control-plane-updates.ts b/web/lib/control-plane-updates.ts index 035558a3..5d312ce7 100644 --- a/web/lib/control-plane-updates.ts +++ b/web/lib/control-plane-updates.ts @@ -1,4 +1,5 @@ import { getSetting, setSetting } from "@/db/queries"; +import { reportServerError } from "@/lib/server-errors"; import { SETTING_KEYS } from "@/lib/settings-keys"; const GITHUB_LATEST_RELEASE_URL = @@ -138,6 +139,9 @@ export async function checkControlPlaneUpdate( channel, }; } catch (error) { + reportServerError(error, "control-plane.update.check", { + tags: { currentVersion }, + }); return { currentVersion, latestVersion: null, @@ -253,6 +257,9 @@ export async function refreshControlPlaneUpgradeState() { try { await checkAndPersistControlPlaneUpdate(upgradeState.targetVersion); } catch (error) { + reportServerError(error, "control-plane.update.refresh-after-upgrade", { + tags: { targetVersion: upgradeState.targetVersion }, + }); console.error( "[control-plane-updates] failed to refresh update state after upgrade", error, @@ -272,6 +279,7 @@ export async function refreshControlPlaneAboutState() { try { await refreshControlPlaneUpgradeState(); } catch (error) { + reportServerError(error, "control-plane.upgrade.refresh"); console.error( "[control-plane-updates] failed to refresh upgrade state on about page", error, diff --git a/web/lib/email/templates/base.tsx b/web/lib/email/templates/base.tsx index ebf66e2c..dd3c2e72 100644 --- a/web/lib/email/templates/base.tsx +++ b/web/lib/email/templates/base.tsx @@ -24,7 +24,7 @@ export function BaseEmail({ preview, children, baseUrl }: BaseEmailProps) { {preview} - +
Techulus Cloud
@@ -47,7 +47,9 @@ const container = { margin: "0 auto", padding: "12px 0 32px", marginBottom: "64px", + width: "100%", maxWidth: "600px", + tableLayout: "fixed" as const, }; const header = { @@ -56,6 +58,9 @@ const header = { const content = { padding: "16px 24px", + overflowWrap: "anywhere" as const, + wordBreak: "break-word" as const, + wordWrap: "break-word" as const, }; const footer = { diff --git a/web/lib/inngest/functions/backup-workflow.ts b/web/lib/inngest/functions/backup-workflow.ts index 9df213a8..c4b4df25 100644 --- a/web/lib/inngest/functions/backup-workflow.ts +++ b/web/lib/inngest/functions/backup-workflow.ts @@ -1,6 +1,7 @@ import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { volumeBackups } from "@/db/schema"; +import { reportOperationFailure } from "@/lib/server-errors"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -71,7 +72,7 @@ export const backupWorkflow = inngest.createFunction( if (!wakeup) { await step.run("handle-backup-timeout", async () => { - await db + const failed = await db .update(volumeBackups) .set({ status: "failed", @@ -82,7 +83,16 @@ export const backupWorkflow = inngest.createFunction( eq(volumeBackups.id, backupId), inArray(volumeBackups.status, ["pending", "uploading"]), ), - ); + ) + .returning({ serviceId: volumeBackups.serviceId }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("backup.failed", { + occurrenceId: backupId, + reason: "timeout", + tags: { backupId, serviceId: failed.serviceId }, + }); + } }); return { status: "failed", reason: "timeout", backupId }; diff --git a/web/lib/inngest/functions/build-workflow.ts b/web/lib/inngest/functions/build-workflow.ts index 16926dd6..ec1207fa 100644 --- a/web/lib/inngest/functions/build-workflow.ts +++ b/web/lib/inngest/functions/build-workflow.ts @@ -3,6 +3,7 @@ import { db } from "@/db"; import { builds, workQueue } from "@/db/schema"; import { deployServiceRevisionInternal } from "@/lib/deploy-service"; import { updatePreviewGitHubStatus } from "@/lib/preview-deployments"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -181,6 +182,9 @@ async function markPreviewBuildFailed( description, }); } catch (error) { + reportServerError(error, "build.preview-status.update", { + tags: { serviceId, revisionId: serviceRevisionId }, + }); console.error("[build-workflow] failed to update preview status:", error); } } @@ -214,13 +218,22 @@ export const buildWorkflow = inngest.createFunction( let groupBuilds = await step.run("get-group-builds", readGroup); if (groupBuilds.length === 0) { - await step.run("report-missing-build-group", () => - markPreviewBuildFailed( + await step.run("report-missing-build-group", async () => { + reportOperationFailure("build.failed", { + occurrenceId: buildGroupId, + reason: "build_group_missing", + tags: { + buildGroupId, + serviceId, + revisionId: serviceRevisionId, + }, + }); + await markPreviewBuildFailed( serviceId, serviceRevisionId, "Preview build group is missing", - ), - ); + ); + }); return { status: "failed", reason: "build_group_missing", buildGroupId }; } if (groupFailure(groupBuilds)) { @@ -251,13 +264,22 @@ export const buildWorkflow = inngest.createFunction( } if (groupBuilds.length === 0) { - await step.run("report-missing-build-group-after-wait", () => - markPreviewBuildFailed( + await step.run("report-missing-build-group-after-wait", async () => { + reportOperationFailure("build.failed", { + occurrenceId: buildGroupId, + reason: "build_group_missing", + tags: { + buildGroupId, + serviceId, + revisionId: serviceRevisionId, + }, + }); + await markPreviewBuildFailed( serviceId, serviceRevisionId, "Preview build group is missing", - ), - ); + ); + }); return { status: "failed", reason: "build_group_missing", buildGroupId }; } if (groupFailure(groupBuilds)) { @@ -272,22 +294,38 @@ export const buildWorkflow = inngest.createFunction( } if (groupBuilds.some((build) => build.status !== "completed")) { await step.run("handle-group-timeout", async () => { - for (const build of groupBuilds) { - if (build.status === "completed") continue; - await db - .update(builds) - .set({ - status: "failed", - error: "Build timed out after 60 minutes", - completedAt: new Date(), - }) - .where( - and( - eq(builds.id, build.id), - inArray(builds.status, nonTerminalBuildStatuses), - ), - ); - } + await Promise.all( + groupBuilds.map(async (build) => { + if (build.status === "completed") return; + const failed = await db + .update(builds) + .set({ + status: "failed", + error: "Build timed out after 60 minutes", + completedAt: new Date(), + }) + .where( + and( + eq(builds.id, build.id), + inArray(builds.status, nonTerminalBuildStatuses), + ), + ) + .returning({ id: builds.id }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("build.failed", { + occurrenceId: failed.id, + reason: "timeout", + tags: { + buildId: failed.id, + buildGroupId, + serviceId, + revisionId: serviceRevisionId, + }, + }); + } + }), + ); }); groupBuilds = await step.run("refresh-group-after-timeout", readGroup); if (groupBuilds.some((build) => build.status !== "completed")) { @@ -317,13 +355,22 @@ export const buildWorkflow = inngest.createFunction( ); } if (!manifest) { - await step.run("report-manifest-timeout", () => - markPreviewBuildFailed( + await step.run("report-manifest-timeout", async () => { + reportOperationFailure("build-manifest.failed", { + occurrenceId: buildGroupId, + reason: "timeout", + tags: { + buildGroupId, + serviceId, + revisionId: serviceRevisionId, + }, + }); + await markPreviewBuildFailed( serviceId, serviceRevisionId, "Preview image manifest timed out", - ), - ); + ); + }); return { status: "completed_no_manifest", buildGroupId }; } if (manifest.status === "failed") { diff --git a/web/lib/inngest/functions/index.ts b/web/lib/inngest/functions/index.ts index dfd95ce4..4a3cab69 100644 --- a/web/lib/inngest/functions/index.ts +++ b/web/lib/inngest/functions/index.ts @@ -29,6 +29,7 @@ export { export { restoreTriggerWorkflow } from "./restore-trigger-workflow"; export { onRestoreFailed, restoreWorkflow } from "./restore-workflow"; export { rolloutWorkflow } from "./rollout-workflow"; +export { sentryFailureWorkflow } from "./sentry-failure-workflow"; export { expiredDeletedServicesPurge, serviceDeletionWorkflow, diff --git a/web/lib/inngest/functions/migration-workflow.ts b/web/lib/inngest/functions/migration-workflow.ts index 9f1a6cc0..88c7e4e1 100644 --- a/web/lib/inngest/functions/migration-workflow.ts +++ b/web/lib/inngest/functions/migration-workflow.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { @@ -10,10 +10,40 @@ import { } from "@/db/schema"; import { deployServiceInternal } from "@/lib/deploy-service"; import { markDeploymentRemoved } from "@/lib/deployment-status"; +import { reportOperationFailure } from "@/lib/server-errors"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; +const activeMigrationStatuses: Array< + NonNullable +> = ["stopping", "backing_up", "deploying_target", "restoring", "starting"]; + +async function markMigrationFailed( + serviceId: string, + failureStage: string, + errorMessage: string, +) { + const failed = await db + .update(services) + .set({ migrationStatus: "failed", migrationError: errorMessage }) + .where( + and( + eq(services.id, serviceId), + inArray(services.migrationStatus, activeMigrationStatuses), + ), + ) + .returning({ id: services.id }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("migration.failed", { + occurrenceId: serviceId, + reason: failureStage, + tags: { serviceId, failureStage }, + }); + } +} + export const migrationWorkflow = inngest.createFunction( { id: "migration-workflow", @@ -178,13 +208,11 @@ export const migrationWorkflow = inngest.createFunction( const backupTimedOut = backupResults.some((r) => r.status === "timed_out"); if (backupTimedOut) { await step.run("handle-backup-timeout", async () => { - await db - .update(services) - .set({ - migrationStatus: "failed", - migrationError: "Backup timed out", - }) - .where(eq(services.id, serviceId)); + await markMigrationFailed( + serviceId, + "backup_timeout", + "Backup timed out", + ); }); return { status: "failed", reason: "backup_timeout" }; } @@ -194,13 +222,11 @@ export const migrationWorkflow = inngest.createFunction( ); if (backupStillPending) { await step.run("handle-backup-still-pending", async () => { - await db - .update(services) - .set({ - migrationStatus: "failed", - migrationError: "Backup did not reach a terminal state", - }) - .where(eq(services.id, serviceId)); + await markMigrationFailed( + serviceId, + "backup_pending", + "Backup did not reach a terminal state", + ); }); return { status: "failed", reason: "backup_pending" }; } @@ -208,13 +234,11 @@ export const migrationWorkflow = inngest.createFunction( const backupFailure = backupResults.find((r) => r.status === "failed"); if (backupFailure) { await step.run("handle-backup-failure", async () => { - await db - .update(services) - .set({ - migrationStatus: "failed", - migrationError: backupFailure.error, - }) - .where(eq(services.id, serviceId)); + await markMigrationFailed( + serviceId, + "backup_failed", + backupFailure.error || "Backup failed", + ); }); return { status: "failed", reason: "backup_failed" }; } @@ -273,13 +297,11 @@ export const migrationWorkflow = inngest.createFunction( const restoreTimedOut = restoreResults.some((r) => r === null); if (restoreTimedOut) { await step.run("handle-restore-timeout", async () => { - await db - .update(services) - .set({ - migrationStatus: "failed", - migrationError: "Restore timed out", - }) - .where(eq(services.id, serviceId)); + await markMigrationFailed( + serviceId, + "restore_timeout", + "Restore timed out", + ); }); return { status: "failed", reason: "restore_timeout" }; } @@ -289,13 +311,11 @@ export const migrationWorkflow = inngest.createFunction( ); if (restoreFailure) { await step.run("handle-restore-failure", async () => { - await db - .update(services) - .set({ - migrationStatus: "failed", - migrationError: restoreFailure.data.error || "Restore failed", - }) - .where(eq(services.id, serviceId)); + await markMigrationFailed( + serviceId, + "restore_failed", + restoreFailure.data.error || "Restore failed", + ); }); return { status: "failed", reason: "restore_failed" }; } diff --git a/web/lib/inngest/functions/on-deployment-failed.ts b/web/lib/inngest/functions/on-deployment-failed.ts index 7fe7e5b3..fcca0fc5 100644 --- a/web/lib/inngest/functions/on-deployment-failed.ts +++ b/web/lib/inngest/functions/on-deployment-failed.ts @@ -58,7 +58,13 @@ export const onDeploymentFailed = inngest.createFunction( ); await step.run("handle-failure", async () => { - await handleRolloutFailure(rolloutId, serviceId, reason, true); + await handleRolloutFailure({ + rolloutId, + serviceId, + reason, + failureStage: "deployment_failed", + isRollingUpdate: true, + }); }); }, ); diff --git a/web/lib/inngest/functions/preview-workflow.ts b/web/lib/inngest/functions/preview-workflow.ts index fc800c30..28672347 100644 --- a/web/lib/inngest/functions/preview-workflow.ts +++ b/web/lib/inngest/functions/preview-workflow.ts @@ -16,6 +16,7 @@ import { deletePreviewService, } from "@/lib/preview-lifecycle"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { reportServerError } from "@/lib/server-errors"; import { pullRequestMergeRef, pullRequestNumberFromMergeRef, @@ -223,16 +224,44 @@ export const previewSyncWorkflow = inngest.createFunction( const previous = await step.run("load-latest-preview-revision", () => loadLatestPreviewRevision(clone.serviceId), ); - let mergeRef: { gitRef: string; sha: string }; + let mergeRef: { gitRef: string; sha: string } | null; try { - mergeRef = await step.run("resolve-merge-ref", () => - resolveGitHubPullRequestMergeRef( - context.githubRepo.installationId, - context.githubRepo.repoFullName, - pullRequestNumber, - ), - ); - } catch { + mergeRef = await step.run("resolve-merge-ref", async () => { + try { + return await resolveGitHubPullRequestMergeRef( + context.githubRepo.installationId, + context.githubRepo.repoFullName, + pullRequestNumber, + ); + } catch (error) { + if ( + error instanceof GitHubApiError && + [404, 409, 422].includes(error.status) + ) { + return null; + } + throw error; + } + }); + } catch (error) { + await step.run("delete-unmergeable-preview", async () => { + reportServerError(error, "preview.merge-ref.resolve", { + tags: { + baseServiceId, + installationId: context.githubRepo.installationId, + pullRequestNumber, + }, + }); + await deletePreviewService( + baseServiceId, + previewGitRef, + "merge ref is unavailable", + ); + }); + return { status: "failed", reason: "merge_ref_unavailable" }; + } + + if (!mergeRef) { await step.run("delete-unmergeable-preview", () => deletePreviewService( baseServiceId, diff --git a/web/lib/inngest/functions/restore-workflow.ts b/web/lib/inngest/functions/restore-workflow.ts index fd6c70b8..3bb77397 100644 --- a/web/lib/inngest/functions/restore-workflow.ts +++ b/web/lib/inngest/functions/restore-workflow.ts @@ -1,3 +1,4 @@ +import { reportOperationFailure } from "@/lib/server-errors"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -30,6 +31,14 @@ export const restoreWorkflow = inngest.createFunction( }); if (!outcome.result) { + await step.run("report-restore-timeout", () => { + reportOperationFailure("restore.failed", { + occurrenceId: backupId, + reason: "timeout", + tags: { backupId }, + }); + return { reported: true }; + }); return { status: "failed", reason: "timeout", backupId }; } diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 926b4a26..2d6b08b0 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -14,6 +14,7 @@ import { getPublishedContainerPorts, type ServiceRevisionSpec, } from "@/lib/service-revision-spec"; +import { reportServerError } from "@/lib/server-errors"; import { assignContainerIp, CONTAINER_IP_ALLOCATION_CONSTRAINTS, @@ -360,6 +361,9 @@ export async function issueCertificatesForRevision( console.log(`[deploy] issued certificate for ${domain}`); issuedDomains.push(domain); } catch (error) { + reportServerError(error, "acme.certificate.issue", { + tags: { domain }, + }); console.error( `[deploy] failed to issue certificate for ${domain}:`, error, diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 065a52d1..6c6d4f4f 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -4,17 +4,38 @@ import { deployments, rollouts } from "@/db/schema"; import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; import { notify } from "@/lib/notifications"; import { updatePreviewGitHubStatus } from "@/lib/preview-deployments"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { enqueueReconcileForAllOnlineServers, enqueueWork, } from "@/lib/work-queue"; -export async function handleRolloutFailure( - rolloutId: string, - serviceId: string, - reason: string, - isRollingUpdate: boolean, -): Promise { +type RolloutFailureStage = + | "workflow_failed" + | "preflight_failed" + | "certificate_provisioning_failed" + | "deployment_failed" + | "health_check_failed" + | "health_check_timeout" + | "dns_sync_timeout"; + +type RolloutFailureOptions = { + rolloutId: string; + serviceId: string; + reason: string; + failureStage: RolloutFailureStage; + isRollingUpdate: boolean; + report?: boolean; +}; + +export async function handleRolloutFailure({ + rolloutId, + serviceId, + reason, + failureStage, + isRollingUpdate, + report = true, +}: RolloutFailureOptions): Promise { const result = await db.transaction(async (tx) => { const [rollout] = await tx .select({ @@ -88,6 +109,20 @@ export async function handleRolloutFailure( if (!result.applied) return; const { rolloutDeployments } = result; const serviceRevisionId = result.rollout.serviceRevisionId; + if (report) { + reportOperationFailure("rollout.failed", { + occurrenceId: rolloutId, + reason: failureStage, + tags: { + rolloutId, + serviceId, + ...(serviceRevisionId ? { revisionId: serviceRevisionId } : {}), + failureStage, + rollbackState: + rolloutDeployments.length === 0 ? "failed" : "rolled_back", + }, + }); + } if (serviceRevisionId) { try { await updatePreviewGitHubStatus({ @@ -97,6 +132,9 @@ export async function handleRolloutFailure( description: `Preview rollout failed: ${reason}`, }); } catch (error) { + reportServerError(error, "rollout.preview-status.update", { + tags: { rolloutId, serviceId, revisionId: serviceRevisionId }, + }); console.error( "[rollout:failure] failed to update preview status:", error, @@ -112,6 +150,9 @@ export async function handleRolloutFailure( serverId: null, failedStage: reason, }).catch((error) => { + reportServerError(error, "rollout.failure.notification", { + tags: { rolloutId, serviceId }, + }); console.error( "[rollout:failure] failed to enqueue deployment failure notification:", error, @@ -129,6 +170,9 @@ export async function handleRolloutFailure( serverId, failedStage: reason, }).catch((error) => { + reportServerError(error, "rollout.failure.notification", { + tags: { rolloutId, serviceId, serverId }, + }); console.error( "[rollout:failure] failed to enqueue deployment failure notification:", error, diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 0c54f5c4..3d4235d5 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -27,6 +27,7 @@ import { } from "@/lib/preview-deployments"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; import { getRolloutServiceRevision } from "@/lib/service-revisions"; +import { reportOperationFailure } from "@/lib/server-errors"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueReconcileForAllOnlineServers } from "@/lib/work-queue"; import { inngest } from "../client"; @@ -197,15 +198,16 @@ export const rolloutWorkflow = inngest.createFunction( if (!rolloutId) return; if (serviceId) { - await handleRolloutFailure( + await handleRolloutFailure({ rolloutId, serviceId, - "workflow_failed", - true, - ); + reason: "workflow_failed", + failureStage: "workflow_failed", + isRollingUpdate: true, + }); } - await db + const fallbackFailure = await db .update(rollouts) .set({ status: "failed", @@ -217,7 +219,27 @@ export const rolloutWorkflow = inngest.createFunction( eq(rollouts.id, rolloutId), inArray(rollouts.status, ["queued", "in_progress"]), ), - ); + ) + .returning({ + serviceId: rollouts.serviceId, + serviceRevisionId: rollouts.serviceRevisionId, + }) + .then((rows) => rows[0]); + if (fallbackFailure) { + reportOperationFailure("rollout.failed", { + occurrenceId: rolloutId, + reason: "workflow_failed", + tags: { + rolloutId, + serviceId: fallbackFailure.serviceId, + ...(fallbackFailure.serviceRevisionId + ? { revisionId: fallbackFailure.serviceRevisionId } + : {}), + failureStage: "workflow_failed", + rollbackState: "failed", + }, + }); + } }, }, async ({ event, step }) => { @@ -257,14 +279,31 @@ export const rolloutWorkflow = inngest.createFunction( if (!acquiredTurn) { await step.run("mark-rollout-queue-timeout", async () => { - await db + const failed = await db .update(rollouts) .set({ status: "failed", currentStage: "queue_timeout", completedAt: new Date(), }) - .where(eq(rollouts.id, rolloutId)); + .where(and(eq(rollouts.id, rolloutId), eq(rollouts.status, "queued"))) + .returning({ serviceRevisionId: rollouts.serviceRevisionId }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("rollout.failed", { + occurrenceId: rolloutId, + reason: "queue_timeout", + tags: { + rolloutId, + serviceId, + ...(failed.serviceRevisionId + ? { revisionId: failed.serviceRevisionId } + : {}), + failureStage: "queue_timeout", + rollbackState: "failed", + }, + }); + } await ingestRolloutLog( rolloutId, serviceId, @@ -337,7 +376,14 @@ export const rolloutWorkflow = inngest.createFunction( "preparing", `Placement validation failed: ${reason}`, ); - await handleRolloutFailure(rolloutId, serviceId, reason, false); + await handleRolloutFailure({ + rolloutId, + serviceId, + reason, + failureStage: "preflight_failed", + isRollingUpdate: false, + report: false, + }); return { success: false as const, reason }; } }); @@ -378,7 +424,14 @@ export const rolloutWorkflow = inngest.createFunction( "preparing", `Placement failed: ${reason}`, ); - await handleRolloutFailure(rolloutId, serviceId, reason, false); + await handleRolloutFailure({ + rolloutId, + serviceId, + reason, + failureStage: "preflight_failed", + isRollingUpdate: false, + report: false, + }); return { success: false as const, reason }; } }); @@ -418,44 +471,69 @@ export const rolloutWorkflow = inngest.createFunction( }); } - const certResult = await step.run("issue-certificates", async () => { - await db - .update(rollouts) - .set({ currentStage: "certificates" }) - .where(eq(rollouts.id, rolloutId)); - try { - const result = await issueCertificatesForRevision(specification); - if (result.issuedDomains.length > 0) { - await ingestRolloutLog( - rolloutId, - serviceId, - "certificates", - `Certificates issued for ${result.issuedDomains.length} domain(s)`, - ); - } - return { success: true as const }; - } catch (error) { - const message = - error instanceof Error - ? error.message - : "Certificate provisioning failed"; - await ingestRolloutLog(rolloutId, serviceId, "certificates", message); - return { success: false as const, reason: message }; + let certificatesIssued = false; + let certificateFailureReason = "Certificate provisioning failed"; + for (let attempt = 1; attempt <= 3; attempt++) { + const certResult = await step.run( + `issue-certificates-${attempt}`, + async () => { + await db + .update(rollouts) + .set({ currentStage: "certificates" }) + .where(eq(rollouts.id, rolloutId)); + try { + const result = await issueCertificatesForRevision(specification); + if (result.issuedDomains.length > 0) { + await ingestRolloutLog( + rolloutId, + serviceId, + "certificates", + `Certificates issued for ${result.issuedDomains.length} domain(s)`, + ); + } + return { success: true as const }; + } catch (error) { + const message = + error instanceof Error + ? error.message + : "Certificate provisioning failed"; + await ingestRolloutLog( + rolloutId, + serviceId, + "certificates", + message, + ); + return { success: false as const, reason: message }; + } + }, + ); + if (certResult.success) { + certificatesIssued = true; + break; } - }); - if (!certResult.success) { + certificateFailureReason = certResult.reason; + if (attempt < 3) { + await step.sleep( + `wait-for-certificate-retry-${attempt}`, + attempt === 1 ? "10s" : "20s", + ); + } + } + + if (!certificatesIssued) { await step.run("handle-certificate-failure", async () => { - await handleRolloutFailure( + await handleRolloutFailure({ rolloutId, serviceId, - "certificate_provisioning_failed", + reason: "certificate_provisioning_failed", + failureStage: "certificate_provisioning_failed", isRollingUpdate, - ); + }); }); return { status: "failed", - reason: certResult.reason, + reason: certificateFailureReason, }; } @@ -582,12 +660,13 @@ export const rolloutWorkflow = inngest.createFunction( ); }); await step.run("handle-health-timeout", async () => { - await handleRolloutFailure( + await handleRolloutFailure({ rolloutId, serviceId, - failedReason, + reason: failedReason, + failureStage: failedReason, isRollingUpdate, - ); + }); }); return { status: "failed", @@ -749,12 +828,13 @@ export const rolloutWorkflow = inngest.createFunction( if (dnsTimedOut) { await step.run("rollback-dns-timeout", async () => { - await handleRolloutFailure( + await handleRolloutFailure({ rolloutId, serviceId, - "dns_sync_timeout", + reason: "dns_sync_timeout", + failureStage: "dns_sync_timeout", isRollingUpdate, - ); + }); }); return { status: "rolled_back", rolloutId, reason: "dns_sync_timeout" }; } diff --git a/web/lib/inngest/functions/sentry-failure-workflow.ts b/web/lib/inngest/functions/sentry-failure-workflow.ts new file mode 100644 index 00000000..8ab7a409 --- /dev/null +++ b/web/lib/inngest/functions/sentry-failure-workflow.ts @@ -0,0 +1,23 @@ +import { inngest } from "@/lib/inngest/client"; +import { reportServerError } from "@/lib/server-errors"; + +export const sentryFailureWorkflow = inngest.createFunction( + { + id: "sentry-function-failure", + triggers: [{ event: "inngest/function.failed" }], + }, + async ({ event }) => { + if (event.data.function_id.endsWith("sentry-function-failure")) { + return; + } + + const error = new Error(event.data.error.message); + error.name = event.data.error.name; + error.stack = event.data.error.stack; + + reportServerError(error, "inngest.function.failed", { + tags: { functionId: event.data.function_id }, + extra: { runId: event.data.run_id }, + }); + }, +); diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index 15bed589..45b2002b 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -36,30 +36,44 @@ import { prepareRegistryArtifactCleanup, } from "@/lib/registry-retention"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; const DELETED_SERVICE_RETENTION_DAYS = 7; -async function markServiceDeletionFailed(serviceId: string, error: unknown) { - await db +async function markServiceOperationFailed( + serviceId: string, + errorMessage: string, + operation: "service-deletion.failed" | "service-restore.failed", + failureStage: string, + expectedStatuses: Array< + NonNullable + >, +) { + const failed = await db .update(services) .set({ deletionStatus: "failed", - deletionError: - error instanceof Error ? error.message : "Service operation failed", + deletionError: errorMessage, }) .where( and( eq(services.id, serviceId), - isNull(services.deletedAt), - or( - eq(services.deletionStatus, "backing_up"), - eq(services.deletionStatus, "deleting"), - ), + inArray(services.deletionStatus, expectedStatuses), ), - ); + ) + .returning({ id: services.id }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure(operation, { + occurrenceId: serviceId, + reason: failureStage, + tags: { serviceId, failureStage }, + }); + } + return Boolean(failed); } export const serviceDeletionWorkflow = inngest.createFunction( @@ -236,17 +250,21 @@ export const serviceDeletionWorkflow = inngest.createFunction( const stillPending = backupResults.some((r) => r.status === "pending"); if (timedOut || failed || stillPending) { await step.run("mark-delete-backup-failed", async () => { - await db - .update(services) - .set({ - deletionStatus: "failed", - deletionError: - failed?.error || - (stillPending - ? "Deletion backup did not reach a terminal state" - : "Deletion backup timed out"), - }) - .where(eq(services.id, serviceId)); + const failureStage = timedOut + ? "backup_timeout" + : stillPending + ? "backup_pending" + : "backup_failed"; + await markServiceOperationFailed( + serviceId, + failed?.error || + (stillPending + ? "Deletion backup did not reach a terminal state" + : "Deletion backup timed out"), + "service-deletion.failed", + failureStage, + ["backing_up"], + ); }); return { status: "failed", @@ -335,7 +353,18 @@ export const serviceDeletionWorkflow = inngest.createFunction( return { status: "deleted", serviceId, backupIds }; } catch (error) { await step.run("mark-unhandled-delete-failed", async () => { - await markServiceDeletionFailed(serviceId, error); + const failed = await markServiceOperationFailed( + serviceId, + error instanceof Error ? error.message : "Service deletion failed", + "service-deletion.failed", + "workflow_failed", + ["backing_up", "deleting"], + ); + if (failed) { + reportServerError(error, "service.deletion.workflow", { + tags: { serviceId }, + }); + } }); return { status: "failed", @@ -504,21 +533,20 @@ export const serviceRestoreWorkflow = inngest.createFunction( const timedOut = restoreResults.some((r) => r.result === null); if (failed || timedOut) { await step.run("mark-restore-failed", async () => { - await db - .update(services) - .set({ - deletionStatus: "failed", - deletionError: - failed?.result?.data.error || "Volume restore timed out", - }) - .where(eq(services.id, serviceId)); + await markServiceOperationFailed( + serviceId, + failed?.result?.data.error || "Volume restore timed out", + "service-restore.failed", + failed ? "volume_restore_failed" : "volume_restore_timeout", + ["restoring"], + ); }); return { status: "failed", reason: failed ? "restore" : "timeout" }; } - const deployResult = await step.run( - "start-restored-deployment", - async () => { + let deployResult: { rolloutId: string }; + try { + deployResult = await step.run("start-restored-deployment", async () => { await db.transaction(async (tx) => { await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, @@ -537,38 +565,62 @@ export const serviceRestoreWorkflow = inngest.createFunction( .where(eq(services.id, serviceId)); }); - try { - const result = await deployServiceInternal(serviceId, actor, { - runtimeBaseRevisionId: setup.runtimeBaseRevisionId, + const result = await deployServiceInternal(serviceId, actor, { + runtimeBaseRevisionId: setup.runtimeBaseRevisionId, + }); + if (!("rolloutId" in result) || !result.rolloutId) { + throw new Error("Restore could not start a deployment"); + } + return { rolloutId: result.rolloutId }; + }); + } catch (error) { + await step.run("mark-restore-deployment-start-failed", async () => { + const failed = await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, + ); + return tx + .update(services) + .set({ + deletedAt: toDate(setup.service.deletedAt), + purgeAfter: toDate(setup.service.purgeAfter), + hostname: null, + originalHostname: setup.service.originalHostname, + deletionStatus: "failed", + deletionError: + error instanceof Error + ? error.message + : "Restore deployment failed", + }) + .where( + and( + eq(services.id, serviceId), + eq(services.deletionStatus, "restoring"), + ), + ) + .returning({ id: services.id }) + .then((rows) => rows[0]); + }); + if (failed) { + reportServerError(error, "service.restore.workflow", { + tags: { serviceId }, }); - if (!("rolloutId" in result) || !result.rolloutId) { - throw new Error("Restore could not start a deployment"); - } - return result; - } catch (error) { - await db.transaction(async (tx) => { - await tx.execute( - sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, - ); - await tx - .update(services) - .set({ - deletedAt: toDate(setup.service.deletedAt), - purgeAfter: toDate(setup.service.purgeAfter), - hostname: null, - originalHostname: setup.service.originalHostname, - deletionStatus: "failed", - deletionError: - error instanceof Error - ? error.message - : "Restore deployment failed", - }) - .where(eq(services.id, serviceId)); + reportOperationFailure("service-restore.failed", { + occurrenceId: serviceId, + reason: "deployment_start_failed", + tags: { + serviceId, + failureStage: "deployment_start_failed", + }, }); - throw error; } - }, - ); + }); + return { + status: "failed", + reason: + error instanceof Error ? error.message : "restore_deploy_failed", + }; + } await group.parallel(() => step.waitForEvent("wait-restore-deployment-status", { @@ -605,11 +657,11 @@ export const serviceRestoreWorkflow = inngest.createFunction( if (!healthyDeployment || failedDeployment) { await step.run("mark-restore-deployment-failed", async () => { - await db.transaction(async (tx) => { + const failed = await db.transaction(async (tx) => { await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, ); - await tx + return tx .update(services) .set({ deletedAt: toDate(setup.service.deletedAt), @@ -621,8 +673,25 @@ export const serviceRestoreWorkflow = inngest.createFunction( failedDeployment?.failedStage || "Restore deployment did not become healthy", }) - .where(eq(services.id, serviceId)); + .where( + and( + eq(services.id, serviceId), + eq(services.deletionStatus, "restoring"), + ), + ) + .returning({ id: services.id }) + .then((rows) => rows[0]); }); + if (failed) { + reportOperationFailure("service-restore.failed", { + occurrenceId: serviceId, + reason: "deployment_unhealthy", + tags: { + serviceId, + failureStage: "deployment_unhealthy", + }, + }); + } }); return { status: "failed", reason: "deployment" }; } @@ -647,7 +716,18 @@ export const serviceRestoreWorkflow = inngest.createFunction( return { status: "restored", serviceId }; } catch (error) { await step.run("mark-unhandled-restore-failed", async () => { - await markServiceDeletionFailed(serviceId, error); + const failed = await markServiceOperationFailed( + serviceId, + error instanceof Error ? error.message : "Service restore failed", + "service-restore.failed", + "workflow_failed", + ["restoring"], + ); + if (failed) { + reportServerError(error, "service.restore.workflow", { + tags: { serviceId }, + }); + } }); return { status: "failed", @@ -729,6 +809,9 @@ export const expiredDeletedServicesPurge = inngest.createFunction( await db.delete(secrets).where(eq(secrets.serviceId, service.id)); await db.delete(services).where(eq(services.id, service.id)); } catch (error) { + reportServerError(error, "service.purge", { + tags: { serviceId: service.id }, + }); console.error( `[service-purge] failed to purge service ${service.id}`, error, diff --git a/web/lib/preview-deployments.ts b/web/lib/preview-deployments.ts index c7d26f61..015a1268 100644 --- a/web/lib/preview-deployments.ts +++ b/web/lib/preview-deployments.ts @@ -84,8 +84,12 @@ async function updatePreviewPullRequestComment(input: { input.previewUrl ? `**Preview:** [Open preview](${input.previewUrl})` : "**Preview:** No public URL configured", - "", - `${escapeGitHubCommentText(input.description.substring(0, 500))}`, + ...(input.state === "failure" + ? [ + "", + `${escapeGitHubCommentText(input.description.substring(0, 500))}`, + ] + : []), ].join("\n"); await upsertGitHubPullRequestComment( input.installationId, diff --git a/web/lib/preview-lifecycle.ts b/web/lib/preview-lifecycle.ts index 22528603..ff7aa5f5 100644 --- a/web/lib/preview-lifecycle.ts +++ b/web/lib/preview-lifecycle.ts @@ -16,6 +16,7 @@ import { cleanupRegistryArtifactsForService, prepareRegistryArtifactCleanup, } from "@/lib/registry-retention"; +import { reportServerError } from "@/lib/server-errors"; import { pullRequestNumberFromMergeRef } from "@/lib/service-revision-spec"; import { enqueueReconcileForAllOnlineServers, @@ -224,6 +225,9 @@ export async function deletePreviewService( description: `Preview removed: ${reason}`, }); } catch (error) { + reportServerError(error, "preview.github-deployments.inactivate", { + tags: { serviceId: claimed.service.id }, + }); console.error( `[preview-lifecycle] failed to inactivate GitHub deployments for ${claimed.service.id}:`, error, diff --git a/web/lib/public-api-routes.ts b/web/lib/public-api-routes.ts index 2ea2f759..af65ee36 100644 --- a/web/lib/public-api-routes.ts +++ b/web/lib/public-api-routes.ts @@ -30,6 +30,7 @@ import { type TimestampCursor, timestampPage, } from "@/lib/public-api-pagination"; +import { reportServerError } from "@/lib/server-errors"; import { queryServiceRevisionChangelog } from "@/lib/service-revision-changelog"; import { isLoggingEnabled, @@ -96,6 +97,7 @@ function cursorFilter( : undefined; } function internalError(error: unknown, operation: string) { + reportServerError(error, `public-api.${operation}`); console.error(`[public-api] ${operation} failed`, error); return apiError("Internal server error", "INTERNAL_ERROR", 500); } @@ -341,6 +343,7 @@ function deployConflict(error: unknown) { ) { return apiError(message, "DEPLOYMENT_CONFLICT", 409); } + reportServerError(error, "public-api.deploy"); return apiError( "Deployment provider unavailable", "DEPLOY_PROVIDER_ERROR", @@ -601,9 +604,11 @@ export async function getServiceLogs( if (error instanceof ServiceLogCursorUnavailableError) { return apiError(error.message, "LOG_CURSOR_UNAVAILABLE", 409); } - return error instanceof RangeError - ? invalidLogQuery(error) - : apiError("Log provider unavailable", "LOG_PROVIDER_ERROR", 502); + if (error instanceof RangeError) return invalidLogQuery(error); + reportServerError(error, "public-api.service-logs.query", { + tags: { serviceId: scope.service.id }, + }); + return apiError("Log provider unavailable", "LOG_PROVIDER_ERROR", 502); } } @@ -737,9 +742,11 @@ export async function getRolloutLogs( })), }); } catch (error) { - return error instanceof RangeError - ? invalidLogQuery(error) - : apiError("Log provider unavailable", "LOG_PROVIDER_ERROR", 502); + if (error instanceof RangeError) return invalidLogQuery(error); + reportServerError(error, "public-api.rollout-logs.query", { + tags: { serviceId: scope.service.id, rolloutId }, + }); + return apiError("Log provider unavailable", "LOG_PROVIDER_ERROR", 502); } } @@ -809,7 +816,10 @@ export async function getMetrics( throwOnError: true, }), }); - } catch { + } catch (error) { + reportServerError(error, "public-api.metrics.query", { + tags: { serviceId: scope.service.id }, + }); return apiError( "Metrics provider unavailable", "METRICS_PROVIDER_ERROR", diff --git a/web/lib/registry-retention.ts b/web/lib/registry-retention.ts index 82c68e4d..ef3cc840 100644 --- a/web/lib/registry-retention.ts +++ b/web/lib/registry-retention.ts @@ -1,6 +1,7 @@ import { and, eq, inArray, isNull, sql } from "drizzle-orm"; import { db } from "@/db"; import { builds, serviceRevisions, workQueue } from "@/db/schema"; +import { reportServerError } from "@/lib/server-errors"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; type RegistryCleanupTransaction = Parameters< @@ -286,6 +287,12 @@ export async function cleanupRegistryArtifactsDaily() { and artifact_deleted_at is null`); }); } catch (error) { + reportServerError(error, "registry.artifact.cleanup", { + tags: { + revisionId: candidate.id, + serviceId: candidate.serviceId, + }, + }); console.error( `[registry-retention] failed to clean artifact ${candidate.id}`, error, diff --git a/web/lib/rollout-enqueue.ts b/web/lib/rollout-enqueue.ts index 9e02daef..ce6e132d 100644 --- a/web/lib/rollout-enqueue.ts +++ b/web/lib/rollout-enqueue.ts @@ -3,6 +3,7 @@ import { db } from "@/db"; import { rollouts } from "@/db/schema"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { reportOperationFailure } from "@/lib/server-errors"; export async function sendRolloutCreated( rolloutId: string, @@ -16,14 +17,31 @@ export async function sendRolloutCreated( ), ); } catch (error) { - await db + const failed = await db .update(rollouts) .set({ status: "failed", currentStage: "enqueue_failed", completedAt: new Date(), }) - .where(and(eq(rollouts.id, rolloutId), eq(rollouts.status, "queued"))); + .where(and(eq(rollouts.id, rolloutId), eq(rollouts.status, "queued"))) + .returning({ serviceRevisionId: rollouts.serviceRevisionId }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("rollout.failed", { + occurrenceId: rolloutId, + reason: "enqueue_failed", + tags: { + rolloutId, + serviceId, + ...(failed.serviceRevisionId + ? { revisionId: failed.serviceRevisionId } + : {}), + failureStage: "enqueue_failed", + rollbackState: "failed", + }, + }); + } throw error; } } diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index b74c627b..0a173f46 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -38,6 +38,7 @@ import { import { notify } from "@/lib/notifications"; import { sendRolloutCreated } from "@/lib/rollout-enqueue"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { AUTOSCALE_ATTEMPT_COOLDOWN_MS, cloneActiveRevisionAndQueueSystemRollout, @@ -227,6 +228,9 @@ export async function runAutoscalingController( created++; await sendRolloutCreated(result.rolloutId, service.id); } catch (error) { + reportServerError(error, "autoscaling.evaluate", { + tags: { serviceId: service.id }, + }); console.error(`[autoscaling] failed to evaluate ${service.name}`, error); } } @@ -354,6 +358,9 @@ export async function rebalanceAutomaticServices( queuedCount++; await sendRolloutCreated(result.rolloutId, service.id); } catch (error) { + reportServerError(error, "scheduler.placement.rebalance", { + tags: { serviceId: service.id }, + }); console.error(`[scheduler] failed to rebalance ${service.name}`, error); } } @@ -490,6 +497,9 @@ export async function recoverInvalidAutomaticPlacements( createdCount++; await sendRolloutCreated(result.rolloutId, serviceId); } catch (error) { + reportServerError(error, "scheduler.placement.recover", { + tags: { serviceId }, + }); console.error( `[scheduler] failed level-triggered recovery for ${serviceDeployments[0]?.serviceName ?? serviceId}`, error, @@ -562,6 +572,13 @@ async function triggerRecoveryForOfflineServers( automaticActiveByService.set(deployment.serviceId, deployment); } } catch (error) { + reportServerError(error, "scheduler.recovery.classify", { + tags: { + deploymentId: deployment.deploymentId, + serviceId: deployment.serviceId, + serverId: deployment.serverId, + }, + }); console.error( `[scheduler] cannot classify deployment ${deployment.deploymentId} for recovery`, error, @@ -587,6 +604,13 @@ async function triggerRecoveryForOfflineServers( createdCount++; await sendRolloutCreated(queued.rolloutId, deployment.serviceId); } catch (error) { + reportServerError(error, "scheduler.recovery.queue", { + tags: { + deploymentId: deployment.deploymentId, + serviceId: deployment.serviceId, + serverId: deployment.serverId, + }, + }); console.error( `[scheduler] automatic recovery failed for ${deployment.serviceName}; periodic recovery will retry`, error, @@ -635,6 +659,9 @@ async function triggerRecoveryForOfflineServers( impactedReplicas: impact.impactedReplicas, serviceNames: [...impact.serviceNames], }).catch((error) => { + reportServerError(error, "scheduler.recovery.notification", { + tags: { serverId }, + }); console.error( `[scheduler] failed to enqueue manual recovery notification for ${impact.serverName}:`, error, @@ -689,6 +716,9 @@ export async function checkAndRecoverStaleServers( serverName: server.name, serverIp: server.wireguardIp || server.publicIp || undefined, }).catch((error) => { + reportServerError(error, "scheduler.server-offline.notification", { + tags: { serverId: server.id }, + }); console.error( `[scheduler] failed to enqueue offline notification for ${server.name}:`, error, @@ -775,6 +805,9 @@ export async function checkAndRunScheduledDeployments(): Promise { `[scheduler] ${service.name}: deployment triggered successfully`, ); } catch (error) { + reportServerError(error, "scheduler.deployment.trigger", { + tags: { serviceId: service.id }, + }); console.error( `[scheduler] failed to process schedule for ${service.name}:`, error, @@ -823,6 +856,13 @@ export async function failTimedOutAgentUpgrades(): Promise { inArray(workQueue.status, ["pending", "processing"]), ), ); + for (const server of timedOut) { + reportOperationFailure("agent-upgrade.failed", { + occurrenceId: server.id, + reason: "timeout", + tags: { serverId: server.id }, + }); + } console.log( `[scheduler] marked ${timedOut.length} agent upgrade(s) timed out`, ); @@ -852,7 +892,11 @@ export async function cleanupStaleItems(): Promise { ), ), ) - .returning({ id: workQueue.id }); + .returning({ + id: workQueue.id, + serverId: workQueue.serverId, + type: workQueue.type, + }); if (items.length > 0) { await tx .update(serviceCommands) @@ -872,6 +916,17 @@ export async function cleanupStaleItems(): Promise { }); if (staleWorkItems.length > 0) { + for (const item of staleWorkItems) { + reportOperationFailure("work-item.failed", { + occurrenceId: item.id, + reason: "lease_expired", + tags: { + serverId: item.serverId, + workItemId: item.id, + workType: item.type, + }, + }); + } console.log( `[scheduler] cleaned up ${staleWorkItems.length} stale work queue items`, ); diff --git a/web/lib/server-errors.ts b/web/lib/server-errors.ts new file mode 100644 index 00000000..ba8275a6 --- /dev/null +++ b/web/lib/server-errors.ts @@ -0,0 +1,45 @@ +import * as Sentry from "@sentry/nextjs"; + +type SafeMetadataValue = string | number | boolean | null; +const MAX_REASON_LENGTH = 500; + +type ReportingContext = { + tags?: Record>; + extra?: Record; +}; + +type OperationFailureContext = ReportingContext & { + occurrenceId: string; + reason?: string; +}; + +export function reportServerError( + error: unknown, + operation: string, + context: ReportingContext = {}, +) { + return Sentry.captureException(error, { + tags: { operation, ...context.tags }, + extra: context.extra, + }); +} + +export function reportOperationFailure( + operation: string, + { occurrenceId, reason, tags, extra }: OperationFailureContext, +) { + const safeReason = reason + // eslint-disable-next-line no-control-regex -- Strip unsafe control characters from reported reasons. + ?.replace(/[\u0000-\u001f\u007f]/g, " ") + .slice(0, MAX_REASON_LENGTH); + return Sentry.captureMessage(`Business operation failed: ${operation}`, { + level: "error", + fingerprint: ["business-failure", operation], + tags: { operation, ...tags }, + extra: { + ...extra, + occurrenceId, + ...(safeReason ? { reason: safeReason } : {}), + }, + }); +} diff --git a/web/lib/service-crons.ts b/web/lib/service-crons.ts index 9b7b2c90..5f0315b5 100644 --- a/web/lib/service-crons.ts +++ b/web/lib/service-crons.ts @@ -7,6 +7,7 @@ import { secrets, serviceCrons, services } from "@/db/schema"; import { decryptSecret } from "@/lib/crypto"; import { notify } from "@/lib/notifications"; import { isSafeCronPath, nextOccurrenceAfter } from "@/lib/public-api"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { ingestCronLog, type CronLog } from "@/lib/victoria-logs"; const MAX_ERROR = 500; @@ -185,6 +186,8 @@ export async function executeServiceCron( let error: string | null = null; let base = ""; let secret: string | undefined; + let failureReason: "configuration_load_failed" | "request_failed" = + "request_failed"; try { const values = await db .select() @@ -204,9 +207,13 @@ export async function executeServiceCron( secret = encrypted.get("CRON_SECRET") ? await decryptSecret(encrypted.get("CRON_SECRET")!) : undefined; - } catch { + } catch (cause) { + reportServerError(cause, "service-cron.configuration.load", { + tags: { cronId, serviceId: row.serviceId }, + }); status = "failed"; error = "Cron configuration could not be loaded"; + failureReason = "configuration_load_failed"; } if (error === null) { try { @@ -258,14 +265,24 @@ export async function executeServiceCron( }; await ingestCronLog(log); if (status === "failed") { + const occurrenceId = cronEventId(cronId, scheduledFor); + reportOperationFailure("service-cron.failed", { + occurrenceId, + reason: failureReason, + tags: { cronId, serviceId: row.serviceId, source }, + extra: { statusCode }, + }); notify({ kind: "cron.failed", - occurrenceId: cronEventId(cronId, scheduledFor), + occurrenceId, serviceId: row.serviceId, path: row.cron.path, statusCode, error, }).catch((cause) => { + reportServerError(cause, "service-cron.notification", { + tags: { cronId, serviceId: row.serviceId }, + }); console.error( "[service-cron] failed to enqueue cron failure notification:", cause, diff --git a/web/lib/victoria-logs.ts b/web/lib/victoria-logs.ts index 540332c8..6297316f 100644 --- a/web/lib/victoria-logs.ts +++ b/web/lib/victoria-logs.ts @@ -10,6 +10,7 @@ import { type EndpointConfig, parseEndpoint, } from "@/lib/victoria"; +import { reportServerError } from "@/lib/server-errors"; const VICTORIA_LOGS_URL = process.env.VICTORIA_LOGS_URL; const VICTORIA_LOGS_PRIVATE_URL = process.env.VICTORIA_LOGS_PRIVATE_URL; @@ -443,6 +444,9 @@ export async function ingestRolloutLog( }, }); } catch (error) { + reportServerError(error, "logs.rollout.ingest", { + tags: { rolloutId, serviceId }, + }); console.error("Failed to ingest rollout log:", error); } } diff --git a/web/lib/victoria-metrics.ts b/web/lib/victoria-metrics.ts index cbe44b92..20360b59 100644 --- a/web/lib/victoria-metrics.ts +++ b/web/lib/victoria-metrics.ts @@ -14,6 +14,7 @@ import { type EndpointConfig, parseEndpoint, } from "@/lib/victoria"; +import { reportServerError } from "@/lib/server-errors"; export { METRIC_RANGE_OPTIONS, type MetricRange, parseMetricRange }; @@ -128,26 +129,27 @@ export async function queryNodeMetricsSnapshots( ): Promise> { const endpoint = getQueryEndpoint(); if (!endpoint) return new Map(); + let reportedError = false; + const recover = (metricName: string) => + queryInstantMetricGroup(endpoint, metricName).catch((error) => { + if (!reportedError) { + reportedError = true; + reportServerError(error, "metrics.nodes.snapshot", { + tags: { metricName }, + }); + } + return new Map(); + }); // 5 total queries (one per metric), results grouped by server_id label. // This collapses the previous N × 5 fan-out. const [cpuMap, memPctMap, memBytesMap, diskPctMap, diskBytesMap] = await Promise.all([ - queryInstantMetricGroup(endpoint, METRIC_NAMES.cpuUsagePercent).catch( - () => new Map(), - ), - queryInstantMetricGroup(endpoint, METRIC_NAMES.memoryUsagePercent).catch( - () => new Map(), - ), - queryInstantMetricGroup(endpoint, METRIC_NAMES.memoryUsedBytes).catch( - () => new Map(), - ), - queryInstantMetricGroup(endpoint, METRIC_NAMES.diskUsagePercent).catch( - () => new Map(), - ), - queryInstantMetricGroup(endpoint, METRIC_NAMES.diskUsedBytes).catch( - () => new Map(), - ), + recover(METRIC_NAMES.cpuUsagePercent), + recover(METRIC_NAMES.memoryUsagePercent), + recover(METRIC_NAMES.memoryUsedBytes), + recover(METRIC_NAMES.diskUsagePercent), + recover(METRIC_NAMES.diskUsedBytes), ]); const result = new Map(); @@ -176,6 +178,7 @@ export async function queryNodeMetricsSnapshot( diskUsagePercent: null, diskUsedBytes: null, }; + let reportedError = false; await Promise.all( Object.entries(METRIC_NAMES).map(async ([key, metricName]) => { @@ -183,7 +186,15 @@ export async function queryNodeMetricsSnapshot( endpoint, metricName, serverId, - ).catch(() => null); + ).catch((error) => { + if (!reportedError) { + reportedError = true; + reportServerError(error, "metrics.node.snapshot", { + tags: { metricName, serverId }, + }); + } + return null; + }); snapshot[key as keyof NodeMetricsSnapshot] = value; }), ); @@ -199,6 +210,7 @@ export async function queryNodeMetricsHistory(options: { }): Promise { const endpoint = getQueryEndpoint(); if (!endpoint) return emptyHistory(); + let reportedError = false; const entries = await Promise.all( Object.entries(METRIC_NAMES).map(async ([key, metricName]) => { @@ -208,7 +220,15 @@ export async function queryNodeMetricsHistory(options: { start: options.start, end: options.end, stepSeconds: options.stepSeconds, - }).catch(() => []); + }).catch((error) => { + if (!reportedError) { + reportedError = true; + reportServerError(error, "metrics.node.history", { + tags: { metricName, serverId: options.serverId }, + }); + } + return []; + }); return [key, series] as const; }), ); @@ -254,8 +274,19 @@ export async function queryServiceMetrics(options: { window.start, window.stepSeconds * SECOND_IN_MILLISECONDS, ); + let reportedError = false; const recover = (promise: Promise, fallback: T): Promise => - options.throwOnError ? promise : promise.catch(() => fallback); + options.throwOnError + ? promise + : promise.catch((error) => { + if (!reportedError) { + reportedError = true; + reportServerError(error, "metrics.service.query", { + tags: { serviceId: options.serviceId }, + }); + } + return fallback; + }); const [ requestResults, diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts index 82b856cf..08f35f99 100644 --- a/web/lib/work-queue.ts +++ b/web/lib/work-queue.ts @@ -12,6 +12,7 @@ import type { WorkQueue } from "@/db/types"; import { MINUTE_IN_MILLISECONDS, subtractMilliseconds } from "@/lib/date"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; import { notifyWorkAvailable } from "@/lib/work-queue-notifications"; export const WORK_QUEUE_MAX_ATTEMPTS = 3; @@ -147,6 +148,9 @@ export async function enqueueWork( await notifyWorkAvailable(serverId, executor); } catch (error) { if (options.tx) throw error; + reportServerError(error, "work-queue.notification.publish", { + tags: { serverId }, + }); console.error("[work-queue] failed to publish notification:", error); } } @@ -271,6 +275,17 @@ export async function completeWorkItemResults( } accepted.push(result.id); + if (result.status === "failed") { + reportOperationFailure("work-item.failed", { + occurrenceId: item.id, + reason: "agent_reported_failure", + tags: { + serverId: item.serverId, + workItemId: item.id, + workType: item.type, + }, + }); + } if (item.type !== "restore_volume") { await runWorkItemCompletionSideEffects(item, result); } @@ -433,6 +448,9 @@ async function markAgentUpgradeStarted(serverId: string, payloadText: string) { ), ); } catch (error) { + reportServerError(error, "agent.upgrade.mark-started", { + tags: { serverId }, + }); console.error("[work-queue] failed to mark agent upgrade started:", error); } } @@ -637,6 +655,9 @@ async function runWorkItemCompletionSideEffects( ); } } catch (error) { + reportServerError(error, "work-queue.completion.side-effects", { + tags: { workItemId: item.id, workType: item.type }, + }); console.error("[work-queue] failed to run completion side effects:", error); } } @@ -650,7 +671,7 @@ async function runAgentUpgradeCompletionSideEffects( if (!payload.targetVersion) return; if (result.status === "failed") { - await db + const failed = await db .update(servers) .set({ agentUpgradeStatus: "failed", @@ -661,7 +682,20 @@ async function runAgentUpgradeCompletionSideEffects( eq(servers.id, item.serverId), eq(servers.agentUpgradeTargetVersion, payload.targetVersion), ), - ); + ) + .returning({ id: servers.id }) + .then((rows) => rows[0]); + if (failed) { + reportOperationFailure("agent-upgrade.failed", { + occurrenceId: item.id, + reason: "agent_reported_failure", + tags: { + serverId: failed.id, + workItemId: item.id, + workType: item.type, + }, + }); + } return; } @@ -688,6 +722,9 @@ async function runAgentUpgradeCompletionSideEffects( ), ); } catch (error) { + reportServerError(error, "agent.upgrade.completion", { + tags: { serverId: item.serverId, workItemId: item.id }, + }); console.error( "[work-queue] failed to run agent upgrade completion side effects:", error, @@ -732,6 +769,9 @@ async function runForceCleanupCompletionSideEffects( ), ); } catch (error) { + reportServerError(error, "work-queue.force-cleanup.completion", { + tags: { serverId: item.serverId, workItemId: item.id }, + }); console.error( "[work-queue] failed to run force cleanup completion side effects:", error, diff --git a/web/tests/agent-backup-failed-route.test.ts b/web/tests/agent-backup-failed-route.test.ts index 42a39ede..d0ea8482 100644 --- a/web/tests/agent-backup-failed-route.test.ts +++ b/web/tests/agent-backup-failed-route.test.ts @@ -24,6 +24,7 @@ const mocks = vi.hoisted(() => { verifyAgentRequest: vi.fn(), revalidatePath: vi.fn(), send: vi.fn(), + reportOperationFailure: vi.fn(), createResourceStatusChanged: vi.fn((data, options) => ({ name: "resource/status.changed", data, @@ -42,6 +43,9 @@ vi.mock("@/lib/inngest/events", () => ({ resourceStatusChanged: { create: mocks.createResourceStatusChanged }, }, })); +vi.mock("@/lib/server-errors", () => ({ + reportOperationFailure: mocks.reportOperationFailure, +})); vi.mock("next/cache", () => ({ revalidatePath: mocks.revalidatePath })); import { POST } from "@/app/api/v1/agent/backup/failed/route"; @@ -74,6 +78,15 @@ describe("agent backup failure", () => { expect(response.status).toBe(200); expect(mocks.revalidatePath).toHaveBeenCalledWith("/dashboard/projects"); + expect(mocks.reportOperationFailure).toHaveBeenCalledWith("backup.failed", { + occurrenceId: "backup-1", + reason: "agent_reported_failure", + tags: { + backupId: "backup-1", + serviceId: "service-1", + serverId: "server-1", + }, + }); expect(mocks.send).toHaveBeenCalledWith({ name: "resource/status.changed", id: "backup-failed-backup-1", @@ -93,6 +106,7 @@ describe("agent backup failure", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ ok: true }); + expect(mocks.reportOperationFailure).not.toHaveBeenCalled(); expect(mocks.revalidatePath).not.toHaveBeenCalled(); expect(mocks.send).not.toHaveBeenCalled(); }); diff --git a/web/tests/build-status-route.test.ts b/web/tests/build-status-route.test.ts index 414afb2f..ee7b7eba 100644 --- a/web/tests/build-status-route.test.ts +++ b/web/tests/build-status-route.test.ts @@ -52,6 +52,8 @@ const mocks = vi.hoisted(() => { updateGitHubDeploymentStatus: vi.fn(), updatePreviewGitHubStatus: vi.fn(), notify: vi.fn(), + reportOperationFailure: vi.fn(), + reportServerError: vi.fn(), createBuildCompleted: vi.fn((data, options) => ({ name: "build/completed", data, @@ -71,6 +73,10 @@ vi.mock("@/lib/github", () => ({ vi.mock("@/lib/preview-deployments", () => ({ updatePreviewGitHubStatus: mocks.updatePreviewGitHubStatus, })); +vi.mock("@/lib/server-errors", () => ({ + reportOperationFailure: mocks.reportOperationFailure, + reportServerError: mocks.reportServerError, +})); vi.mock("@/lib/work-queue", () => ({ enqueueWork: mocks.enqueueWork })); vi.mock("@/lib/inngest/client", () => ({ inngest: { send: mocks.send } })); vi.mock("@/lib/inngest/events", () => ({ @@ -181,6 +187,26 @@ describe("agent build status transitions", () => { buildId: "build-amd64", error: undefined, }); + expect(mocks.reportOperationFailure).toHaveBeenCalledWith("build.failed", { + occurrenceId: "build-amd64", + reason: "agent_reported_failure", + tags: { + buildId: "build-amd64", + serviceId: "service-1", + revisionId: "revision-1", + serverId: "server-1", + }, + }); + }); + + it("does not report a replayed failed transition", async () => { + const failedBuild = build("failed"); + mocks.selectResults.push([failedBuild], [{ specification }], [failedBuild]); + mocks.updateResults.push([]); + + expect((await post("failed")).status).toBe(200); + expect(mocks.reportOperationFailure).not.toHaveBeenCalled(); + expect(mocks.notify).not.toHaveBeenCalled(); }); it("keeps the service details link on GitHub deployment statuses", async () => { diff --git a/web/tests/deploy-service-revision.test.ts b/web/tests/deploy-service-revision.test.ts index f13b2505..ff24bc5f 100644 --- a/web/tests/deploy-service-revision.test.ts +++ b/web/tests/deploy-service-revision.test.ts @@ -8,7 +8,12 @@ const mocks = vi.hoisted(() => ({ startMigrationInternal: vi.fn(), triggerBuildInternal: vi.fn(), send: vi.fn(), - updateWhere: vi.fn(), + updateWhere: vi.fn(() => ({ + returning: vi.fn(() => + Promise.resolve([{ serviceRevisionId: "revision-1" }]), + ), + })), + reportOperationFailure: vi.fn(), createRolloutCreated: vi.fn((data, options) => ({ name: "rollout/created", data, @@ -38,6 +43,9 @@ vi.mock("@/lib/service-revisions", () => ({ createRolloutWithServiceRevision: mocks.createRolloutWithServiceRevision, })); vi.mock("@/lib/inngest/client", () => ({ inngest: { send: mocks.send } })); +vi.mock("@/lib/server-errors", () => ({ + reportOperationFailure: mocks.reportOperationFailure, +})); vi.mock("@/lib/trigger-build", () => ({ triggerBuildInternal: mocks.triggerBuildInternal, })); diff --git a/web/tests/email-templates.test.tsx b/web/tests/email-templates.test.tsx new file mode 100644 index 00000000..993e30d5 --- /dev/null +++ b/web/tests/email-templates.test.tsx @@ -0,0 +1,38 @@ +import { render } from "@react-email/render"; +import { describe, expect, it } from "vitest"; +import { Alert } from "@/lib/email/templates/alert"; +import { MemberInvitation } from "@/lib/email/templates/member-invitation"; + +const longContent = "x".repeat(2_000); + +const templates = { + alert: Alert({ + bannerText: "BUILD FAILED", + heading: "Build Failure Alert", + description: "The build failed.", + details: [{ label: "Error", value: longContent }], + }), + invitation: MemberInvitation({ + inviterName: longContent, + role: "member", + inviteUrl: `https://cloud.example.com/invite/${longContent}`, + }), +}; + +describe("email templates", () => { + it.each(Object.entries(templates))( + "constrains and wraps long %s content", + async (_, template) => { + const html = await render(template); + + expect(html).toContain(longContent); + expect(html).toContain('width="600"'); + expect(html).toContain("width:100%"); + expect(html).toContain("max-width:600px"); + expect(html).toContain("table-layout:fixed"); + expect(html).toContain("overflow-wrap:anywhere"); + expect(html).toContain("word-break:break-word"); + expect(html).toContain("word-wrap:break-word"); + }, + ); +}); diff --git a/web/tests/inngest-route.test.ts b/web/tests/inngest-route.test.ts index 881d0295..d3991297 100644 --- a/web/tests/inngest-route.test.ts +++ b/web/tests/inngest-route.test.ts @@ -32,6 +32,7 @@ const mocks = vi.hoisted(() => { rolloutWorkflow: { id: "rollout-workflow" }, scheduledBackupsCheck: { id: "scheduled-backups-check" }, scheduledDeploymentsCheck: { id: "scheduled-deployments-check" }, + sentryFailureWorkflow: { id: "sentry-function-failure" }, serviceCommandRetention: { id: "service-command-retention" }, serviceCronDispatcher: { id: "service-cron-dispatcher" }, serviceCronWorkflow: { id: "service-cron-workflow" }, diff --git a/web/tests/preview-deployments.test.ts b/web/tests/preview-deployments.test.ts index 8e6efd0f..f4108b77 100644 --- a/web/tests/preview-deployments.test.ts +++ b/web/tests/preview-deployments.test.ts @@ -404,6 +404,9 @@ describe("preview service cloning", () => { }), ); expect(mocks.upsertGitHubPullRequestComment).toHaveBeenCalledTimes(2); + expect( + mocks.upsertGitHubPullRequestComment.mock.calls[0]?.[4], + ).not.toContain(""); expect(mocks.upsertGitHubPullRequestComment).toHaveBeenLastCalledWith( 101, "acme/app", diff --git a/web/tests/preview-workflow.test.ts b/web/tests/preview-workflow.test.ts index 859c36a8..4619d67a 100644 --- a/web/tests/preview-workflow.test.ts +++ b/web/tests/preview-workflow.test.ts @@ -39,6 +39,7 @@ const mocks = vi.hoisted(() => { deletePreviewService: vi.fn(), triggerResolvedBuildInternal: vi.fn(), parseServiceRevisionSpec: vi.fn(), + reportServerError: vi.fn(), send: vi.fn(), createSyncEvent: vi.fn((data, options) => ({ name: "preview/sync-requested", @@ -66,6 +67,9 @@ vi.mock("@/lib/preview-lifecycle", () => ({ vi.mock("@/lib/service-revision-changes", () => ({ parseServiceRevisionSpec: mocks.parseServiceRevisionSpec, })); +vi.mock("@/lib/server-errors", () => ({ + reportServerError: mocks.reportServerError, +})); vi.mock("@/lib/trigger-build", () => ({ triggerResolvedBuildInternal: mocks.triggerResolvedBuildInternal, })); @@ -198,7 +202,38 @@ describe("preview lifecycle workflows", () => { }); }); - it("deletes the preview when GitHub has no merge ref", async () => { + it.each([404, 409, 422])( + "deletes the preview without reporting when GitHub returns %i for the merge ref", + async (status) => { + mocks.selectResults.push( + [baseContext], + [{ previewOfService: "base-service" }], + [], + ); + mocks.resolveGitHubPullRequestMergeRef.mockRejectedValue( + new mocks.GitHubApiError("merge ref unavailable", status), + ); + + await expect( + invoke(previewSyncWorkflow, { + baseServiceId: "base-service", + previewGitRef: "refs/pull/42/merge", + }), + ).resolves.toEqual({ + status: "failed", + reason: "merge_ref_unavailable", + }); + expect(mocks.deletePreviewService).toHaveBeenCalledWith( + "base-service", + "refs/pull/42/merge", + "merge ref is unavailable", + ); + expect(mocks.reportServerError).not.toHaveBeenCalled(); + expect(mocks.triggerResolvedBuildInternal).not.toHaveBeenCalled(); + }, + ); + + it("reports an unexpected merge-ref failure once before cleanup", async () => { mocks.selectResults.push( [baseContext], [{ previewOfService: "base-service" }], @@ -222,6 +257,18 @@ describe("preview lifecycle workflows", () => { "refs/pull/42/merge", "merge ref is unavailable", ); + expect(mocks.reportServerError).toHaveBeenCalledOnce(); + expect(mocks.reportServerError).toHaveBeenCalledWith( + expect.any(Error), + "preview.merge-ref.resolve", + { + tags: { + baseServiceId: "base-service", + installationId: 10, + pullRequestNumber: 42, + }, + }, + ); expect(mocks.triggerResolvedBuildInternal).not.toHaveBeenCalled(); }); diff --git a/web/tests/rollout-certificate-retry.test.ts b/web/tests/rollout-certificate-retry.test.ts new file mode 100644 index 00000000..5651f3fb --- /dev/null +++ b/web/tests/rollout-certificate-retry.test.ts @@ -0,0 +1,189 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + function updateQuery() { + const query = { + set: vi.fn(() => query), + where: vi.fn(async () => undefined), + }; + return query; + } + + return { + update: vi.fn(updateQuery), + issueCertificatesForRevision: vi.fn(), + handleRolloutFailure: vi.fn(), + ingestRolloutLog: vi.fn(), + }; +}); + +vi.mock("@/db", () => ({ db: { update: mocks.update } })); +vi.mock("@/db/queries", () => ({ getService: vi.fn() })); +vi.mock("@/lib/deployment-status", () => ({ + isObservedReady: vi.fn(), + observedReadyPhases: [], +})); +vi.mock("@/lib/preview-deployments", () => ({ + canDeployServiceRevision: vi.fn(), + updatePreviewGitHubStatus: vi.fn(), +})); +vi.mock("@/lib/routing-sync", () => ({ buildRoutingTargets: vi.fn() })); +vi.mock("@/lib/service-revisions", () => ({ + getRolloutServiceRevision: vi.fn(), +})); +vi.mock("@/lib/victoria-logs", () => ({ + ingestRolloutLog: mocks.ingestRolloutLog, +})); +vi.mock("@/lib/work-queue", () => ({ + enqueueReconcileForAllOnlineServers: vi.fn(), +})); +vi.mock("@/lib/inngest/client", () => ({ + inngest: { + createFunction: vi.fn( + (_options: unknown, handler: (input: unknown) => unknown) => handler, + ), + }, +})); +vi.mock("@/lib/inngest/events", () => ({ + inngestEvents: { + rolloutCreated: { name: "rollout/created" }, + rolloutCancelled: { name: "rollout/cancelled" }, + resourceStatusChanged: { name: "resource/status.changed" }, + serverDnsSynced: { name: "server/dns.synced" }, + }, +})); +vi.mock("@/lib/inngest/functions/rollout-helpers", () => ({ + checkForRollingUpdate: vi.fn(), + cleanupExistingDeployments: vi.fn(), + cleanupTerminalDeployments: vi.fn(), + completeRollout: vi.fn(), + createDeploymentRecords: vi.fn(), + issueCertificatesForRevision: mocks.issueCertificatesForRevision, + resolveRevisionPlacements: vi.fn(), + validateServers: vi.fn(), +})); +vi.mock("@/lib/inngest/functions/rollout-utils", () => ({ + handleRolloutFailure: mocks.handleRolloutFailure, +})); + +import { rolloutWorkflow } from "@/lib/inngest/functions/rollout-workflow"; + +function invokeRollout() { + const step = { + run: vi.fn(async (name: string, operation: () => unknown) => { + if ( + name.startsWith("issue-certificates-") || + name === "handle-certificate-failure" + ) { + return operation(); + } + + switch (name) { + case "validate-service": + return false; + case "acquire-rollout-turn-0": + return "acquired"; + case "load-service-revision": + return { + id: "revision-1", + specification: { + ports: [], + serverless: { enabled: false }, + }, + }; + case "log-rollout-started": + return undefined; + case "load-placements": + return { + success: true, + placements: [{ serverId: "server-1", replicas: 1 }], + totalReplicas: 1, + }; + case "validate-servers": + return { success: true, serverIds: ["server-1"] }; + case "cleanup-terminal-deployments": + return undefined; + case "check-rolling-update": + return false; + case "cleanup-existing": + return undefined; + case "create-deployments": + throw new Error("continued after certificates"); + default: + throw new Error(`unexpected step: ${name}`); + } + }), + sleep: vi.fn(async () => undefined), + waitForEvent: vi.fn(), + }; + const handler = rolloutWorkflow as unknown as (input: { + event: { data: { rolloutId: string; serviceId: string } }; + step: typeof step; + }) => Promise; + + return { + result: handler({ + event: { + data: { rolloutId: "rollout-1", serviceId: "service-1" }, + }, + step, + }), + step, + }; +} + +describe("rollout certificate retries", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses durable backoff and continues after a later attempt succeeds", async () => { + mocks.issueCertificatesForRevision + .mockRejectedValueOnce(new Error("first outage")) + .mockRejectedValueOnce(new Error("second outage")) + .mockResolvedValueOnce({ issuedDomains: [] }); + + const { result, step } = invokeRollout(); + + await expect(result).rejects.toThrow("continued after certificates"); + expect(mocks.issueCertificatesForRevision).toHaveBeenCalledTimes(3); + expect(step.sleep).toHaveBeenNthCalledWith( + 1, + "wait-for-certificate-retry-1", + "10s", + ); + expect(step.sleep).toHaveBeenNthCalledWith( + 2, + "wait-for-certificate-retry-2", + "20s", + ); + expect(step.run).toHaveBeenCalledWith( + "create-deployments", + expect.any(Function), + ); + expect(mocks.handleRolloutFailure).not.toHaveBeenCalled(); + }); + + it("uses the existing failure path once after three failed attempts", async () => { + mocks.issueCertificatesForRevision + .mockRejectedValueOnce(new Error("first outage")) + .mockRejectedValueOnce(new Error("second outage")) + .mockRejectedValueOnce(new Error("final outage")); + + const { result, step } = invokeRollout(); + + await expect(result).resolves.toEqual({ + status: "failed", + reason: "final outage", + }); + expect(mocks.issueCertificatesForRevision).toHaveBeenCalledTimes(3); + expect(step.sleep).toHaveBeenCalledTimes(2); + expect(mocks.handleRolloutFailure).toHaveBeenCalledOnce(); + expect(mocks.handleRolloutFailure).toHaveBeenCalledWith( + "rollout-1", + "service-1", + "certificate_provisioning_failed", + false, + ); + }); +}); diff --git a/web/tests/sentry-failure-workflow.test.ts b/web/tests/sentry-failure-workflow.test.ts new file mode 100644 index 00000000..f1908ab9 --- /dev/null +++ b/web/tests/sentry-failure-workflow.test.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + reportServerError: vi.fn(), +})); + +vi.mock("@/lib/inngest/client", () => ({ + inngest: { + createFunction: vi.fn( + (_options: unknown, handler: (input: unknown) => unknown) => handler, + ), + }, +})); +vi.mock("@/lib/server-errors", () => ({ + reportServerError: mocks.reportServerError, +})); + +import { sentryFailureWorkflow } from "@/lib/inngest/functions/sentry-failure-workflow"; + +type FailureEvent = { + data: { + error: { message: string; name: string; stack: string }; + function_id: string; + run_id: string; + }; +}; + +function invoke(event: FailureEvent) { + return ( + sentryFailureWorkflow as unknown as (input: { + event: FailureEvent; + }) => Promise + )({ event }); +} + +describe("Sentry Inngest failure workflow", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("ignores failures from itself", async () => { + await invoke({ + data: { + error: { message: "report failed", name: "Error", stack: "stack" }, + function_id: "techulus-cloud-sentry-function-failure", + run_id: "run-self", + }, + }); + + expect(mocks.reportServerError).not.toHaveBeenCalled(); + }); + + it("reports another function after retries are exhausted", async () => { + await invoke({ + data: { + error: { message: "build failed", name: "BuildError", stack: "stack" }, + function_id: "techulus-cloud-build-workflow", + run_id: "run-build", + }, + }); + + expect(mocks.reportServerError).toHaveBeenCalledOnce(); + expect(mocks.reportServerError).toHaveBeenCalledWith( + expect.objectContaining({ + message: "build failed", + name: "BuildError", + stack: "stack", + }), + "inngest.function.failed", + { + tags: { functionId: "techulus-cloud-build-workflow" }, + extra: { runId: "run-build" }, + }, + ); + }); +}); diff --git a/web/tests/server-errors.test.ts b/web/tests/server-errors.test.ts new file mode 100644 index 00000000..d9041b12 --- /dev/null +++ b/web/tests/server-errors.test.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + captureMessage: vi.fn(), +})); + +vi.mock("@sentry/nextjs", () => mocks); + +import { reportOperationFailure, reportServerError } from "@/lib/server-errors"; + +describe("server error reporting", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("reports caught exceptions with operation metadata", () => { + const error = new Error("provider unavailable"); + + reportServerError(error, "github.repositories.list", { + tags: { installationId: "installation-1" }, + extra: { attempt: 2 }, + }); + + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + operation: "github.repositories.list", + installationId: "installation-1", + }, + extra: { attempt: 2 }, + }); + }); + + it("groups business failures by operation", () => { + reportOperationFailure("build.failed", { + occurrenceId: "build-1", + reason: "Build timed out", + tags: { serverId: "server-1" }, + extra: { timeoutSeconds: 300 }, + }); + + expect(mocks.captureMessage).toHaveBeenCalledWith( + "Business operation failed: build.failed", + { + level: "error", + fingerprint: ["business-failure", "build.failed"], + tags: { + operation: "build.failed", + serverId: "server-1", + }, + extra: { + timeoutSeconds: 300, + occurrenceId: "build-1", + reason: "Build timed out", + }, + }, + ); + }); + + it("bounds and strips control characters from business reasons", () => { + reportOperationFailure("work-item.failed", { + occurrenceId: "work-1", + reason: `failed\n${"x".repeat(600)}`, + }); + + expect(mocks.captureMessage).toHaveBeenCalledWith( + "Business operation failed: work-item.failed", + expect.objectContaining({ + extra: { + occurrenceId: "work-1", + reason: `failed ${"x".repeat(493)}`, + }, + }), + ); + }); +}); diff --git a/web/tests/service-crons.test.ts b/web/tests/service-crons.test.ts index ba144ddf..24c83122 100644 --- a/web/tests/service-crons.test.ts +++ b/web/tests/service-crons.test.ts @@ -1,8 +1,55 @@ import { EventEmitter } from "node:events"; import type { ClientRequest, IncomingMessage, RequestOptions } from "node:http"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const selectResults: unknown[][] = []; + const updateResults: unknown[][] = []; + function query(result: unknown[]) { + const value = { + from: vi.fn(() => value), + innerJoin: vi.fn(() => value), + where: vi.fn(() => value), + limit: vi.fn(() => value), + set: vi.fn(() => value), + returning: vi.fn(() => value), + // oxlint-disable-next-line unicorn/no-thenable -- Drizzle query builders are awaitable. + then: ( + resolve: (rows: unknown[]) => unknown, + reject?: (reason: unknown) => unknown, + ) => Promise.resolve(result).then(resolve, reject), + }; + return value; + } + return { + selectResults, + updateResults, + db: { + select: vi.fn(() => query(selectResults.shift() ?? [])), + update: vi.fn(() => query(updateResults.shift() ?? [])), + }, + decryptSecret: vi.fn(), + notify: vi.fn(), + reportOperationFailure: vi.fn(), + reportServerError: vi.fn(), + ingestCronLog: vi.fn(), + }; +}); + +vi.mock("@/db", () => ({ db: mocks.db })); +vi.mock("@/lib/crypto", () => ({ decryptSecret: mocks.decryptSecret })); +vi.mock("@/lib/notifications", () => ({ notify: mocks.notify })); +vi.mock("@/lib/server-errors", () => ({ + reportOperationFailure: mocks.reportOperationFailure, + reportServerError: mocks.reportServerError, +})); +vi.mock("@/lib/victoria-logs", () => ({ + ingestCronLog: mocks.ingestCronLog, +})); + import { cronEventId, + executeServiceCron, latestDueOccurrence, nextOccurrenceAfter, parseCronUrl, @@ -11,6 +58,14 @@ import { } from "@/lib/service-crons"; describe("service cron scheduling and requests", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.selectResults.length = 0; + mocks.updateResults.length = 0; + mocks.notify.mockResolvedValue(undefined); + mocks.ingestCronLog.mockResolvedValue(undefined); + }); + it("returns the first UTC occurrence strictly after the supplied instant", () => { expect( nextOccurrenceAfter( @@ -131,6 +186,58 @@ describe("service cron scheduling and requests", () => { vi.useRealTimers(); } }); + + it("reports a claimed failed execution without URL or secret context", async () => { + const scheduledFor = new Date("2026-08-06T10:05:00Z"); + mocks.selectResults.push( + [ + { + cron: { + schedule: "* * * * *", + path: "/private/job", + }, + serviceId: "service-1", + }, + ], + [ + { + key: "CRON_BASE_URL", + encryptedValue: "encrypted-sensitive-value", + }, + ], + ); + mocks.updateResults.push([{ id: "cron-1" }], []); + mocks.decryptSecret.mockRejectedValue(new Error("decrypt failed")); + + await expect( + executeServiceCron("cron-1", "* * * * *", scheduledFor, "scheduled"), + ).resolves.toMatchObject({ stale: false, status: "failed" }); + + expect(mocks.reportOperationFailure).toHaveBeenCalledWith( + "service-cron.failed", + { + occurrenceId: cronEventId("cron-1", scheduledFor), + reason: "configuration_load_failed", + tags: { + cronId: "cron-1", + serviceId: "service-1", + source: "scheduled", + }, + extra: { statusCode: null }, + }, + ); + expect(mocks.reportServerError).toHaveBeenCalledWith( + expect.any(Error), + "service-cron.configuration.load", + { tags: { cronId: "cron-1", serviceId: "service-1" } }, + ); + const captured = JSON.stringify([ + mocks.reportOperationFailure.mock.calls, + mocks.reportServerError.mock.calls, + ]); + expect(captured).not.toContain("/private/job"); + expect(captured).not.toContain("encrypted-sensitive-value"); + }); }); function fakeRequest( diff --git a/web/tests/work-queue.test.ts b/web/tests/work-queue.test.ts index 8cd2e66c..d9075930 100644 --- a/web/tests/work-queue.test.ts +++ b/web/tests/work-queue.test.ts @@ -77,6 +77,8 @@ const mocks = vi.hoisted(() => { execute: vi.fn().mockResolvedValue({ rows: [] }), }, send: vi.fn(), + reportOperationFailure: vi.fn(), + reportServerError: vi.fn(), }; }); @@ -109,6 +111,10 @@ vi.mock("@/lib/inngest/events", () => ({ }, }, })); +vi.mock("@/lib/server-errors", () => ({ + reportOperationFailure: mocks.reportOperationFailure, + reportServerError: mocks.reportServerError, +})); vi.mock("@/lib/work-queue-notifications", () => ({ notifyWorkAvailable: vi.fn(), })); @@ -161,6 +167,8 @@ beforeEach(() => { mocks.db.execute.mockClear(); mocks.send.mockReset(); mocks.send.mockResolvedValue(undefined); + mocks.reportOperationFailure.mockReset(); + mocks.reportServerError.mockReset(); }); describe("command work completion", () => { @@ -317,6 +325,35 @@ describe("restore work completion", () => { error: "checksum mismatch", }, }); + expect(mocks.reportOperationFailure).toHaveBeenCalledWith( + "work-item.failed", + { + occurrenceId: "work-1", + reason: "agent_reported_failure", + tags: { + serverId: "server-1", + workItemId: "work-1", + workType: "restore_volume", + }, + }, + ); + expect( + JSON.stringify(mocks.reportOperationFailure.mock.calls), + ).not.toContain("checksum mismatch"); + }); + + it("does not report a replayed failed result", async () => { + await completeWorkItemResults("server-1", [ + { id: "work-1", attempt: 1, status: "failed", error: "first failure" }, + ]); + mocks.reportOperationFailure.mockClear(); + + const replay = await completeWorkItemResults("server-1", [ + { id: "work-1", attempt: 1, status: "failed", error: "first failure" }, + ]); + + expect(replay.accepted).toEqual([]); + expect(mocks.reportOperationFailure).not.toHaveBeenCalled(); }); it("publishes the terminal migration event from persisted context", async () => {