From b66177ce66c87a54636aa765c3848a54d0cc9942 Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Sun, 30 Aug 2026 08:40:15 -0400 Subject: [PATCH 1/5] fix(worker): reindex repos with missing zoekt shards If INDEX_CACHE_DIR is wiped independently of the DB (e.g. placed on ephemeral storage in Kubernetes), repos stay marked as indexed while their shard files are gone, and search silently returns nothing until the next scheduled reindex. On startup, scan the index directory and re-queue any repo the DB believes is indexed but has no shard file on disk, reusing the existing repo-index workload and its per-repo execution lock. --- packages/backend/src/index.ts | 3 +- .../backend/src/repoCleanupWorkload.test.ts | 164 +++++++++++++++++- packages/backend/src/repoCleanupWorkload.ts | 72 +++++++- 3 files changed, 236 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index fc2ec78fc..f3636bc9d 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,7 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { redis } from "./redis.js"; import { createConnectionSyncWorkload } from "./connectionSyncWorkload.js"; -import { cleanupOrphanedRepoResources, createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoCleanupWorkload, reindexReposWithMissingShards } from "./repoCleanupWorkload.js"; import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; @@ -93,6 +93,7 @@ jobManager.register(auditLogPruneWorkload); const api = new Api(promClient, prisma, jobManager, redis, settings); await cleanupOrphanedRepoResources(prisma); +await reindexReposWithMissingShards(prisma, jobManager); const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); await configManager.syncConfig(); diff --git a/packages/backend/src/repoCleanupWorkload.test.ts b/packages/backend/src/repoCleanupWorkload.test.ts index 4a1e770e5..43519b4d7 100644 --- a/packages/backend/src/repoCleanupWorkload.test.ts +++ b/packages/backend/src/repoCleanupWorkload.test.ts @@ -1,6 +1,8 @@ import type { PrismaClient } from "@sourcebot/db"; +import { JOB_PRIORITIES } from "@sourcebot/shared"; import { beforeEach, describe, expect, test, vi } from "vitest"; -import { createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { createRepoCleanupWorkload, reindexReposWithMissingShards } from "./repoCleanupWorkload.js"; +import type { JobManager } from "./types.js"; const fsMocks = vi.hoisted(() => ({ existsSync: vi.fn(), @@ -30,12 +32,14 @@ vi.mock("fs/promises", () => ({ })); const repoFindUnique = vi.fn(); +const repoFindMany = vi.fn(); const repoDeleteMany = vi.fn(); const repoUpdate = vi.fn(); const db = { repo: { findUnique: repoFindUnique, + findMany: repoFindMany, deleteMany: repoDeleteMany, update: repoUpdate, }, @@ -77,6 +81,7 @@ describe("repoCleanupWorkload", () => { fsMocks.readdir.mockResolvedValue([]); fsMocks.rm.mockResolvedValue(undefined); repoFindUnique.mockResolvedValue(eligibleRepo); + repoFindMany.mockResolvedValue([]); repoDeleteMany.mockResolvedValue({ count: 1 }); repoUpdate.mockResolvedValue(undefined); }); @@ -183,3 +188,160 @@ describe("repoCleanupWorkload", () => { ); }); }); + +describe("reindexReposWithMissingShards", () => { + const trigger = vi.fn(); + const jobManager = { trigger } as unknown as JobManager; + + beforeEach(() => { + vi.clearAllMocks(); + fsMocks.existsSync.mockReturnValue(true); + fsMocks.readdir.mockResolvedValue([]); + repoFindMany.mockResolvedValue([]); + trigger.mockResolvedValue("job-id"); + }); + + test("does nothing when the index directory doesn't exist", async () => { + fsMocks.existsSync.mockReturnValue(false); + + await reindexReposWithMissingShards(db, jobManager); + + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(repoFindMany).not.toHaveBeenCalled(); + expect(trigger).not.toHaveBeenCalled(); + }); + + test("re-queues an indexed repo with no shard on disk", async () => { + fsMocks.readdir.mockResolvedValue([]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + // Pins down the exact where-clause: repos still eligible for reindex + // scheduling (has a connection, or explicitly pinned via + // isAutoCleanupDisabled) that the DB believes are indexed. This mirrors + // the set reconcileJobSchedulers.ts keeps on a recurring reindex + // schedule, since orphaned repos with no such pin are the cleanup + // workload's responsibility, not this one's. + expect(repoFindMany).toHaveBeenCalledWith({ + where: { + indexedAt: { not: null }, + OR: [ + { connections: { some: {} } }, + { isAutoCleanupDisabled: true }, + ], + }, + select: { id: true, name: true }, + }); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + + test("does not re-queue a repo that already has a shard on disk", async () => { + fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).not.toHaveBeenCalled(); + }); + + test("treats a lingering .tmp shard as missing", async () => { + fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt.tmp"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + + test("ignores unrelated files in the index directory", async () => { + fsMocks.readdir.mockResolvedValue([".DS_Store", "README.md"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + + test("recognizes a repo whose content is split across multiple shard files", async () => { + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_42_v16.00001.zoekt", + ]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).not.toHaveBeenCalled(); + }); + + test("only re-queues the repo actually missing a shard among many", async () => { + fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/healthy-repo" }, + { id: 43, name: "github.com/acme/broken-repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).toHaveBeenCalledTimes(1); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 43 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + + test("re-queues remaining repos even if one fails to enqueue", async () => { + fsMocks.readdir.mockResolvedValue([]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/flaky-repo" }, + { id: 43, name: "github.com/acme/broken-repo" }, + ]); + trigger.mockImplementation(async (_name, data: { repoId: number }) => { + if (data.repoId === 42) { + throw new Error("redis connection reset"); + } + return "job-id"; + }); + + await expect( + reindexReposWithMissingShards(db, jobManager), + ).resolves.not.toThrow(); + + expect(trigger).toHaveBeenCalledTimes(2); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 43 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + expect(lifecycleLogger.error).toHaveBeenCalledWith( + expect.stringContaining( + "Failed to re-queue repo github.com/acme/flaky-repo (id: 42)", + ), + expect.any(Error), + ); + }); +}); diff --git a/packages/backend/src/repoCleanupWorkload.ts b/packages/backend/src/repoCleanupWorkload.ts index d0fedbe81..997ce7882 100644 --- a/packages/backend/src/repoCleanupWorkload.ts +++ b/packages/backend/src/repoCleanupWorkload.ts @@ -3,13 +3,14 @@ import { createLogger, getRepoIdFromPath, getRepoPath, + JOB_PRIORITIES, REPO_CLEANUP_QUEUE, } from "@sourcebot/shared"; import { existsSync } from "fs"; import { readdir, rm } from "fs/promises"; import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from "./constants.js"; import { REPOSITORY_EXECUTION_LOCK } from "./repoLock.js"; -import type { Settings, Workload } from "./types.js"; +import type { JobManager, Settings, Workload } from "./types.js"; import { getRepoIdFromShardFileName } from "./utils.js"; const logger = createLogger("repo-cleanup-workload"); @@ -231,3 +232,72 @@ export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { } } }; + +// Handles the inverse of cleanupOrphanedRepoResources: repos the DB believes are +// indexed but whose shard files are missing from disk (e.g., INDEX_CACHE_DIR was +// wiped independently of the DB, as happens when it's placed on ephemeral storage). +// Without this, such repos would silently return empty search results until their +// next scheduled reindex, which can be a long time away. +export const reindexReposWithMissingShards = async ( + db: PrismaClient, + jobManager: JobManager, +) => { + if (!existsSync(INDEX_CACHE_DIR)) { + return; + } + + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdsWithShards = new Set(); + for (const entry of entries) { + // .tmp files are left behind by in-progress or previously failed index + // attempts. They aren't searchable, so they don't count as a valid shard. + if (entry.includes(".tmp")) { + continue; + } + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + repoIdsWithShards.add(repoId); + } + } + + // Considers the same set of repos reconcileJobSchedulers keeps on a recurring + // reindex schedule: attached to a connection, or explicitly pinned via + // isAutoCleanupDisabled. Anything outside that set is owned by the cleanup + // workload above, not re-indexed. + const indexedRepos = await db.repo.findMany({ + where: { + indexedAt: { not: null }, + OR: [ + { connections: { some: {} } }, + { isAutoCleanupDisabled: true }, + ], + }, + select: { id: true, name: true }, + }); + + const reposMissingShards = indexedRepos.filter( + (repo) => !repoIdsWithShards.has(repo.id), + ); + + // Triggered sequentially so that one repo failing to enqueue (e.g. a + // transient Redis error) doesn't stop the rest from being recovered, and + // can't take down startup: this runs before the worker installs its + // uncaught-exception handlers. + for (const repo of reposMissingShards) { + logger.warn( + `Repo ${repo.name} (id: ${repo.id}) is marked as indexed but has no shard files on disk. Re-queuing for indexing.`, + ); + try { + await jobManager.trigger( + "repo-index", + { repoId: repo.id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + } catch (error) { + logger.error( + `Failed to re-queue repo ${repo.name} (id: ${repo.id}) for indexing:`, + error, + ); + } + } +}; From c2af5348ce166e57743ab52a9aa80819edadb831 Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Sun, 30 Aug 2026 08:41:04 -0400 Subject: [PATCH 2/5] chore: add changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46c363645..92ccb9b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Reindexed repositories on startup when their persisted indexed state no longer had corresponding Zoekt shard files on disk. [#1621](https://github.com/sourcebot-dev/sourcebot/pull/1621) + ## [5.1.10] - 2026-08-27 ### Fixed From 5acdcbdbd3beaf78b4e146c3b32359850caeda7a Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Sun, 30 Aug 2026 09:05:08 -0400 Subject: [PATCH 3/5] fix(worker): handle missing zoekt index directory Two gaps in the missing-shard reconciliation added for #1210: - A missing INDEX_CACHE_DIR caused an early return, skipping recovery entirely for the exact scenario the check exists to handle (the whole index directory gone). Now treated as zero shards on disk instead. - Shard detection accepted any numeric-prefixed filename, so the .meta sidecar zoekt writes alongside every shard, or an unrelated file, could make a repo look healthy with no searchable index. Now requires the real .zoekt suffix. --- .../backend/src/repoCleanupWorkload.test.ts | 62 ++++++++++++++++++- packages/backend/src/repoCleanupWorkload.ts | 19 ++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/repoCleanupWorkload.test.ts b/packages/backend/src/repoCleanupWorkload.test.ts index 43519b4d7..e01e53084 100644 --- a/packages/backend/src/repoCleanupWorkload.test.ts +++ b/packages/backend/src/repoCleanupWorkload.test.ts @@ -201,14 +201,21 @@ describe("reindexReposWithMissingShards", () => { trigger.mockResolvedValue("job-id"); }); - test("does nothing when the index directory doesn't exist", async () => { + test("still recovers eligible repos when the index directory doesn't exist", async () => { fsMocks.existsSync.mockReturnValue(false); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); await reindexReposWithMissingShards(db, jobManager); expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(repoFindMany).not.toHaveBeenCalled(); - expect(trigger).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalled(); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); }); test("re-queues an indexed repo with no shard on disk", async () => { @@ -253,6 +260,22 @@ describe("reindexReposWithMissingShards", () => { expect(trigger).not.toHaveBeenCalled(); }); + test("does not re-queue a repo whose shard and .meta sidecar are both present", async () => { + // The normal healthy state: zoekt always writes the .meta sidecar + // alongside the real shard, so both show up in the same readdir(). + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_42_v16.00000.zoekt.meta", + ]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).not.toHaveBeenCalled(); + }); + test("treats a lingering .tmp shard as missing", async () => { fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt.tmp"]); repoFindMany.mockResolvedValue([ @@ -268,6 +291,39 @@ describe("reindexReposWithMissingShards", () => { ); }); + test("treats the .meta sidecar file alone as missing", async () => { + // zoekt writes a `.meta` file alongside every real shard. If + // only the sidecar survives a partial wipe, the repo has no searchable + // index and must still be re-queued. + fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt.meta"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + + test("does not treat a numeric-prefixed non-shard file as a valid shard", async () => { + fsMocks.readdir.mockResolvedValue(["1_42_backup"]); + repoFindMany.mockResolvedValue([ + { id: 42, name: "github.com/acme/repo" }, + ]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { repoId: 42 }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + }); + test("ignores unrelated files in the index directory", async () => { fsMocks.readdir.mockResolvedValue([".DS_Store", "README.md"]); repoFindMany.mockResolvedValue([ diff --git a/packages/backend/src/repoCleanupWorkload.ts b/packages/backend/src/repoCleanupWorkload.ts index 997ce7882..6b6ef66d0 100644 --- a/packages/backend/src/repoCleanupWorkload.ts +++ b/packages/backend/src/repoCleanupWorkload.ts @@ -242,16 +242,23 @@ export const reindexReposWithMissingShards = async ( db: PrismaClient, jobManager: JobManager, ) => { - if (!existsSync(INDEX_CACHE_DIR)) { - return; + // A missing directory means zero shards exist, not that there's nothing to + // recover: it's the same "everything is gone" scenario this function exists + // to handle, so it must still fall through to the DB lookup below. + let entries: string[]; + if (existsSync(INDEX_CACHE_DIR)) { + entries = await readdir(INDEX_CACHE_DIR); + } else { + entries = []; } - const entries = await readdir(INDEX_CACHE_DIR); const repoIdsWithShards = new Set(); for (const entry of entries) { - // .tmp files are left behind by in-progress or previously failed index - // attempts. They aren't searchable, so they don't count as a valid shard. - if (entry.includes(".tmp")) { + // Only a real, searchable shard file counts. This excludes in-progress + // or failed .tmp artifacts, the .meta sidecar zoekt writes alongside + // each shard, and any other numeric-prefixed file that isn't actually + // an index (e.g. a stray backup file). + if (!entry.endsWith(".zoekt")) { continue; } const repoId = getRepoIdFromShardFileName(entry); From 2c5e1fb40497356b1e4ce34b0ddb32b82bc9237c Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Sun, 30 Aug 2026 10:02:42 -0400 Subject: [PATCH 4/5] fix(worker): recover shards after config sync Missing-shard recovery ran before configManager.syncConfig(), so it could evaluate repo eligibility against connections about to be removed by this startup's config sync, wasting a reindex enqueue on a repo that's about to be orphaned. Moved it after syncConfig(), which handles connection removal synchronously, and still before reconcileJobSchedulers()/jobManager.start(). --- packages/backend/src/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index f3636bc9d..91e32240e 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -93,11 +93,18 @@ jobManager.register(auditLogPruneWorkload); const api = new Api(promClient, prisma, jobManager, redis, settings); await cleanupOrphanedRepoResources(prisma); -await reindexReposWithMissingShards(prisma, jobManager); const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); await configManager.syncConfig(); +// Runs after config sync so a repo whose connection this sync just removed +// (handled synchronously in syncConfig) isn't wrongly re-queued right before +// it's orphaned. Connections added or changed by this sync are applied +// asynchronously by their own connection-sync job, which can't run until +// jobManager.start() below, so that side of eligibility is unaffected by +// this ordering either way. +await reindexReposWithMissingShards(prisma, jobManager); + await reconcileJobSchedulers({ db: prisma, jobManager, From c761c2f36c2da11d81e8fc23355c59e13ca4a379 Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Tue, 1 Sep 2026 13:30:59 -0400 Subject: [PATCH 5/5] fix(worker): don't let a transient disk/DB error crash startup in reindexReposWithMissingShards readdir(INDEX_CACHE_DIR) and db.repo.findMany() ran unprotected, contradicting the function's own comment that a failure here "can't take down startup" - an unhandled rejection there would crash the process before jobManager.start() or reconcileJobSchedulers() ever run. Wrap the detection phase in a single try/catch that logs and returns early, leaving missed repos to recover on their next scheduled reindex. The per-repo trigger loop's existing isolation is untouched. Adds 3 tests: no-op with zero indexed repos, and the two new failure paths resolving without throwing. --- .../backend/src/repoCleanupWorkload.test.ts | 41 ++++++++ packages/backend/src/repoCleanupWorkload.ts | 93 +++++++++++-------- 2 files changed, 95 insertions(+), 39 deletions(-) diff --git a/packages/backend/src/repoCleanupWorkload.test.ts b/packages/backend/src/repoCleanupWorkload.test.ts index e01e53084..1836e5b6d 100644 --- a/packages/backend/src/repoCleanupWorkload.test.ts +++ b/packages/backend/src/repoCleanupWorkload.test.ts @@ -201,6 +201,47 @@ describe("reindexReposWithMissingShards", () => { trigger.mockResolvedValue("job-id"); }); + test("no-op when there are no indexed repos, even if stray shard-like files exist on disk", async () => { + fsMocks.readdir.mockResolvedValue(["1_42_v16.00000.zoekt"]); + repoFindMany.mockResolvedValue([]); + + await reindexReposWithMissingShards(db, jobManager); + + expect(trigger).not.toHaveBeenCalled(); + }); + + test("logs and returns instead of throwing when the DB lookup fails", async () => { + // Regression: this detection phase runs before the worker installs its + // uncaught-exception handlers (see repoCleanupWorkload.ts), so a transient + // failure here must degrade to "recovery skipped this run", not an unhandled + // rejection that crashes startup. + repoFindMany.mockRejectedValue(new Error("connection reset")); + + await expect( + reindexReposWithMissingShards(db, jobManager), + ).resolves.not.toThrow(); + + expect(trigger).not.toHaveBeenCalled(); + expect(lifecycleLogger.error).toHaveBeenCalledWith( + expect.stringContaining("Failed to detect repos with missing shard files"), + expect.any(Error), + ); + }); + + test("logs and returns instead of throwing when reading the index directory fails", async () => { + fsMocks.readdir.mockRejectedValue(new Error("EACCES: permission denied")); + + await expect( + reindexReposWithMissingShards(db, jobManager), + ).resolves.not.toThrow(); + + expect(trigger).not.toHaveBeenCalled(); + expect(lifecycleLogger.error).toHaveBeenCalledWith( + expect.stringContaining("Failed to detect repos with missing shard files"), + expect.any(Error), + ); + }); + test("still recovers eligible repos when the index directory doesn't exist", async () => { fsMocks.existsSync.mockReturnValue(false); repoFindMany.mockResolvedValue([ diff --git a/packages/backend/src/repoCleanupWorkload.ts b/packages/backend/src/repoCleanupWorkload.ts index 6b6ef66d0..76e4a765f 100644 --- a/packages/backend/src/repoCleanupWorkload.ts +++ b/packages/backend/src/repoCleanupWorkload.ts @@ -242,49 +242,64 @@ export const reindexReposWithMissingShards = async ( db: PrismaClient, jobManager: JobManager, ) => { - // A missing directory means zero shards exist, not that there's nothing to - // recover: it's the same "everything is gone" scenario this function exists - // to handle, so it must still fall through to the DB lookup below. - let entries: string[]; - if (existsSync(INDEX_CACHE_DIR)) { - entries = await readdir(INDEX_CACHE_DIR); - } else { - entries = []; - } - - const repoIdsWithShards = new Set(); - for (const entry of entries) { - // Only a real, searchable shard file counts. This excludes in-progress - // or failed .tmp artifacts, the .meta sidecar zoekt writes alongside - // each shard, and any other numeric-prefixed file that isn't actually - // an index (e.g. a stray backup file). - if (!entry.endsWith(".zoekt")) { - continue; + // This whole detection phase (disk scan + DB lookup) is wrapped in one try/catch + // so a transient failure here (e.g. a disk read error or a DB hiccup) degrades to + // "recovery skipped this run" instead of an unhandled rejection — this runs before + // the worker installs its uncaught-exception handlers, so an unhandled rejection + // here would crash startup, contradicting the resilience the per-repo loop below + // already has. + let reposMissingShards: { id: number; name: string }[]; + try { + // A missing directory means zero shards exist, not that there's nothing to + // recover: it's the same "everything is gone" scenario this function exists + // to handle, so it must still fall through to the DB lookup below. + let entries: string[]; + if (existsSync(INDEX_CACHE_DIR)) { + entries = await readdir(INDEX_CACHE_DIR); + } else { + entries = []; } - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - repoIdsWithShards.add(repoId); + + const repoIdsWithShards = new Set(); + for (const entry of entries) { + // Only a real, searchable shard file counts. This excludes in-progress + // or failed .tmp artifacts, the .meta sidecar zoekt writes alongside + // each shard, and any other numeric-prefixed file that isn't actually + // an index (e.g. a stray backup file). + if (!entry.endsWith(".zoekt")) { + continue; + } + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + repoIdsWithShards.add(repoId); + } } - } - // Considers the same set of repos reconcileJobSchedulers keeps on a recurring - // reindex schedule: attached to a connection, or explicitly pinned via - // isAutoCleanupDisabled. Anything outside that set is owned by the cleanup - // workload above, not re-indexed. - const indexedRepos = await db.repo.findMany({ - where: { - indexedAt: { not: null }, - OR: [ - { connections: { some: {} } }, - { isAutoCleanupDisabled: true }, - ], - }, - select: { id: true, name: true }, - }); + // Considers the same set of repos reconcileJobSchedulers keeps on a recurring + // reindex schedule: attached to a connection, or explicitly pinned via + // isAutoCleanupDisabled. Anything outside that set is owned by the cleanup + // workload above, not re-indexed. + const indexedRepos = await db.repo.findMany({ + where: { + indexedAt: { not: null }, + OR: [ + { connections: { some: {} } }, + { isAutoCleanupDisabled: true }, + ], + }, + select: { id: true, name: true }, + }); - const reposMissingShards = indexedRepos.filter( - (repo) => !repoIdsWithShards.has(repo.id), - ); + reposMissingShards = indexedRepos.filter( + (repo) => !repoIdsWithShards.has(repo.id), + ); + } catch (error) { + logger.error( + "Failed to detect repos with missing shard files; skipping recovery for this startup. They'll be re-queued on their next scheduled reindex.", + error, + ); + return; + } // Triggered sequentially so that one repo failing to enqueue (e.g. a // transient Redis error) doesn't stop the rest from being recovered, and