Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

Declarative connection configuration#235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brendan-kellam merged 10 commits into from
Mar 21, 2025
Merged

Declarative connection configuration #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ebb2544
wip on introducing declarative configs
brendan-kellam Mar 20, 2025
9604210
Add the actual config loading
brendan-kellam Mar 20, 2025
2cdd327
Add concept of 'service error wrapper'
brendan-kellam Mar 20, 2025
b45ceda
Add error handling for duplicate connection names
brendan-kellam Mar 20, 2025
6182bd2
(fix) Gate /connections and /settings route on authentication. This h…
brendan-kellam Mar 20, 2025
b586488
add environment variable support for declarative configs
brendan-kellam Mar 21, 2025
038eca3
Add settings configuration
brendan-kellam Mar 21, 2025
8f2ddfa
remove unused types
brendan-kellam Mar 21, 2025
47075bb
Remove SOURCEBOT_ROOT_DOMAIN since we can get that info from the headers
brendan-kellam Mar 21, 2025
e180555
add schema validation for declarative configs
brendan-kellam Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .env.development
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
Expand DownExpand Up@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"

# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"

SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
Expand All@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=

# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",

"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",

"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},
Expand Down
1 change: 1 addition & 0 deletions packages/backend/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/backend/src/env.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),

INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitea.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');

export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;

const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,
Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/github.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}

export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;

const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
Expand All@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o

if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/gitlab.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;

const api = new Gitlab({
...(token ? {
Expand Down
2 changes: 0 additions & 2 deletions packages/backend/src/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});

type Arguments = {
configPath: string;
cacheDir: string;
}

Expand All@@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}

const prisma = new PrismaClient();
Expand Down
43 changes: 39 additions & 4 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";

const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});

const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}

const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();

const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}

return {
...DEFAULT_SETTINGS,
...config.settings,
}
}

export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
Expand All@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});

const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);

const promClient = new PromClient();

Expand Down
10 changes: 5 additions & 5 deletions packages/backend/src/repoManager.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";

interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
Expand DownExpand Up@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {

const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
Expand DownExpand Up@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);

const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;

Expand All@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}

const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;

Expand All@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}

this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

Expand Down
47 changes: 3 additions & 44 deletions packages/backend/src/types.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";

export type AppContext = {
/**
* Path to the repos cache directory.
Expand All@@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;

cachePath: string;

configPath: string;
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;

/**
* Structure of the `metadata` field in the `Repo` table.
Expand Down
65 changes: 35 additions & 30 deletions packages/backend/src/utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}

export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
});

if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}

const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
return envToken;
}
}

export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}

export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
Expand Down
7 changes: 3 additions & 4 deletions packages/backend/src/zoekt.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
Expand All@@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";

const logger = createLogger('zoekt');

export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
Expand DownExpand Up@@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}

const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;

return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
Expand Down
Loading