feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

feat(web): Improved repository table#572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 16 commits into from
Oct 25, 2025
Merged

feat(web): Improved repository table #572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7212640
wip on new & improved repos table
brendan-kellam Oct 22, 2025
49578b1
remove old repos table
brendan-kellam Oct 22, 2025
e5bb570
more cleanup
brendan-kellam Oct 23, 2025
d86119f
padding nits
brendan-kellam Oct 23, 2025
d6ca03c
changelog
brendan-kellam Oct 23, 2025
58d6da7
move config settings getter to shared package
brendan-kellam Oct 23, 2025
39bcb09
use threshold date for job failure timeout
brendan-kellam Oct 23, 2025
cd54eb0
more touchups
brendan-kellam Oct 23, 2025
8168ec2
Store HEAD commit hash in DB. Also improve how we display dates
brendan-kellam Oct 24, 2025
662ebee
fixed column width
brendan-kellam Oct 24, 2025
f76fd0c
s
brendan-kellam Oct 24, 2025
c16ef08
tooltip for commit hash
brendan-kellam Oct 24, 2025
5a670bf
Add branch table to repos detail view
brendan-kellam Oct 25, 2025
2b51f96
fix build
brendan-kellam Oct 25, 2025
357cc09
Merge branch 'main' into bkellam/repos_table_v2
brendan-kellam Oct 25, 2025
a206eaa
fix build
brendan-kellam Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)

### Removed
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
Expand Down
21 changes: 0 additions & 21 deletions packages/backend/src/constants.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,6 @@
import { env } from "./env.js";
import { Settings } from "./types.js";
import path from "path";

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
maxTrigramCount: 20000,
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second
reindexRepoPollingIntervalMs: 1000 * 1, // 1 second
maxConnectionSyncJobConcurrency: 8,
maxRepoIndexingJobConcurrency: 8,
maxRepoGarbageCollectionJobConcurrency: 8,
repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead
experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours
}

export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
'github',
];
Expand Down
22 changes: 22 additions & 0 deletions packages/backend/src/git.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -268,4 +268,26 @@ export const getTags = async (path: string) => {
const git = createGitClientForPath(path);
const tags = await git.tags();
return tags.all;
}

export const getCommitHashForRefName = async ({
path,
refName,
}: {
path: string,
refName: string,
}) => {
const git = createGitClientForPath(path);

try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;

// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}
21 changes: 3 additions & 18 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,12 +2,12 @@ import "./instrument.js";

import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement, loadConfig } from '@sourcebot/shared';
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
import { ConnectionManager } from './connectionManager.js';
import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
Expand All@@ -18,20 +18,6 @@ import { PromClient } from './promClient.js';

const logger = createLogger('backend-entrypoint');

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const config = await loadConfig(configPath);

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}


const reposPath = REPOS_CACHE_DIR;
const indexPath = INDEX_CACHE_DIR;

Expand All@@ -57,8 +43,7 @@ redis.ping().then(() => {

const promClient = new PromClient();

const settings = await getSettings(env.CONFIG_PATH);

const settings = await getConfigSettings(env.CONFIG_PATH);

if (hasEntitlement('github-app')) {
await GithubAppManager.getInstance().init(prisma);
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/repoCompileUtils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';

export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

Expand Down
86 changes: 79 additions & 7 deletions packages/backend/src/repoIndexManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';

Expand DownExpand Up@@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
}: {}),
});

this.worker.on('completed', this.onJobCompleted.bind(this));
Expand DownExpand Up@@ -126,7 +129,7 @@ export class RepoIndexManager {
{
AND: [
{ status: RepoIndexingJobStatus.FAILED },
{ completedAt: { gt: timeoutDate } },
{ completedAt: { gt: thresholdDate } },
]
}
]
Expand DownExpand Up@@ -263,7 +266,16 @@ export class RepoIndexManager {

try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);

await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
Expand All@@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
Expand DownExpand Up@@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}

let revisions = [
'HEAD'
];

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);

return revisions;
}

private async cleanupRepository(repo: Repo, logger: Logger) {
Expand All@@ -384,16 +440,32 @@ export class RepoIndexManager {
data: {
status: RepoIndexingJobStatus.COMPLETED,
completedAt: new Date(),
},
include: {
repo: true,
}
});

const jobTypeLabel = getJobTypePrometheusLabel(jobData.type);

if (jobData.type === RepoIndexingJobType.INDEX) {
const { path: repoPath } = getRepoPath(jobData.repo);
const commitHash = await getCommitHashForRefName({
path: repoPath,
refName: 'HEAD',
});

const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);

const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

Expand Down
28 changes: 0 additions & 28 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";

export type Settings = Required<SettingsSchema>;

// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),

/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),

/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});

export type RepoMetadata = z.infer<typeof repoMetadataSchema>;

// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;
Expand Down
54 changes: 4 additions & 50 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];

export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);

if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);

revisions = [
...revisions,
...matchingBranches
];
}

if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);

revisions = [
...revisions,
...matchingTags
];
}

// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}

const command = [
'zoekt-git-index',
'-allow_missing_branches',
Expand All@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
reject(error);
return;
}

if (stdout) {
stdout.split('\n').filter(line => line.trim()).forEach(line => {
logger.info(line);
Expand All@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
logger.info(line);
});
}

resolve({
stdout,
stderr
Expand Down
Loading
Loading