Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

Migrate to postgres#174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msukkari merged 5 commits into from
Jan 23, 2025
Merged

Migrate to postgres #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eb8788e
spin up postgres in docker container
msukkari Jan 22, 2025
845937e
get initial pol of postgres db working in docker image
msukkari Jan 22, 2025
f00a4c0
spin up postgres server in dev case
msukkari Jan 23, 2025
079a2dd
updated syncedAt date after config sync:
msukkari Jan 23, 2025
b4a90d7
remove unnecessary port expose in docker file
msukkari Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions Dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,7 +47,6 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK

RUN yarn workspace @sourcebot/web build


# ------ Build Backend ------
FROM node-alpine AS backend-builder
WORKDIR /app
Expand All@@ -59,7 +58,8 @@ COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
RUN yarn workspace @sourcebot/backend build



# ------ Runner ------
FROM node-alpine AS runner
WORKDIR /app
Expand All@@ -68,6 +68,8 @@ ENV NEXT_TELEMETRY_DISABLED=1
ENV DATA_DIR=/data
ENV CONFIG_PATH=$DATA_DIR/config.json
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
ENV DB_DATA_DIR=$DATA_CACHE_DIR/db
ENV DB_NAME=sourcebot

ARG SOURCEBOT_VERSION=unknown
ENV SOURCEBOT_VERSION=$SOURCEBOT_VERSION
Expand All@@ -94,23 +96,23 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK
# ENV SOURCEBOT_TELEMETRY_DISABLED=1

# Configure dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib

# Configure zoekt
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
RUN mkdir -p ${DATA_CACHE_DIR}
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/

# Configure the webapp
COPY --from=web-builder /app/packages/web/public ./packages/web/public
Expand All@@ -122,6 +124,9 @@ COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend

# Configure the database
RUN mkdir -p /run/postgresql && \
chown -R postgres:postgres /run/postgresql && \
chmod 775 /run/postgresql
COPY --from=database-builder /app/node_modules ./node_modules
COPY --from=database-builder /app/packages/db ./packages/db

Expand Down
32 changes: 28 additions & 4 deletions entrypoint.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,10 +19,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
mkdir -p "$DATA_CACHE_DIR"
fi

# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
yarn workspace @sourcebot/db prisma:migrate:prod
# Check if DB_DATA_DIR exists, if not initialize it
if [ ! -d "$DB_DATA_DIR" ]; then
echo -e "\e[34m[Info] Initializing database at $DB_DATA_DIR...\e[0m"
mkdir -p $DB_DATA_DIR && chown -R postgres:postgres "$DB_DATA_DIR"
su postgres -c "initdb -D $DB_DATA_DIR"
fi

# In order to detect if this is the first run, we create a `.installed` file in
# the cache directory.
Expand DownExpand Up@@ -162,5 +164,27 @@ fi
# }


# Start the database and wait for it to be ready before starting any other service
su postgres -c "postgres -D $DB_DATA_DIR" &
until pg_isready -h localhost -p 5432 -U postgres; do
echo -e "\e[34m[Info] Waiting for the database to be ready...\e[0m"
sleep 1
done

# Check if the database already exists, and create it if it dne
EXISTING_DB=$(psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'")

if [ "$EXISTING_DB" = "1" ]; then
echo "Database '$DB_NAME' already exists; skipping creation."
else
echo "Creating database '$DB_NAME'..."
psql -U postgres -c "CREATE DATABASE \"$DB_NAME\""
fi

# Run a Database migration
export DATABASE_URL="postgresql://postgres@localhost:5432/$DB_NAME"
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod

# Run supervisord
exec supervisord -c /etc/supervisor/conf.d/supervisord.conf
14 changes: 10 additions & 4 deletions package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,13 +6,19 @@
"scripts": {
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:mt": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web dev:redis",
"dev": "cross-env SOURCEBOT_TENANT_MODE=single npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:mt": "cross-env SOURCEBOT_TENANT_MODE=multi npm-run-all --print-label dev:deps:start dev:deps:wait dev:start",
"dev:start": "yarn workspace @sourcebot/db prisma:migrate:dev && cross-env npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web dev:redis",
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn workspace @sourcebot/web dev",
"dev:redis": "docker ps --filter \"name=redis\" --format \"{{.Names}}\" | grep -q \"^redis$\" && docker rm -f redis; docker run -d --name redis -p 6379:6379 redis"
"dev:deps:start": "npm-run-all --print-label --parallel dev:postgres dev:redis",
"dev:deps:wait": "npm-run-all --print-label --parallel dev:postgres:wait dev:redis:wait",
"dev:deps:stop": "docker stop redis postgres",
"dev:redis": "docker ps --filter 'name=redis' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name redis -p 6379:6379 redis:7.4.2",
"dev:redis:wait": "sh -c 'while ! nc -z localhost 6379; do echo \"Waiting for Redis...\"; sleep 1; done'",
"dev:postgres": "docker ps --filter 'name=postgres' --filter 'status=running' --quiet | grep -q . || docker run -d --rm --name postgres -p 5432:5432 -v $(pwd)/.sourcebot/db:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sourcebot postgres:17.2",
"dev:postgres:wait": "sh -c 'while ! nc -z localhost 5432; do echo \"Waiting for PostgreSQL...\"; sleep 1; done'"
},
"devDependencies": {
"cross-env": "^7.0.3",
Expand Down
1 change: 1 addition & 0 deletions packages/backend/src/main.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -223,6 +223,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
},
data: {
syncStatus: ConfigSyncStatus.SYNCED,
syncedAt: new Date()
}
})
});
Expand Down
2 changes: 1 addition & 1 deletion packages/db/.env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
DATABASE_URL=file:../../../.sourcebot/db.sqlite
DATABASE_URL=postgresql://postgres@localhost:5432/sourcebot
18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250114182749_repo_init/migration.sql
View file
Open in desktop

This file was deleted.

30 changes: 0 additions & 30 deletions packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

45 changes: 0 additions & 45 deletions packages/db/prisma/migrations/20250115193735_auth_js_models/migration.sql
View file
Open in desktop

This file was deleted.

25 changes: 0 additions & 25 deletions packages/db/prisma/migrations/20250115221611_add_indexing_status_to_repo/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250116233531_add_orgs_table/migration.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions packages/db/prisma/migrations/20250117183646_add_config_table/migration.sql
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions packages/db/prisma/migrations/20250120201905_add_role/migration.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions packages/db/prisma/migrations/20250120222015_add_active_org_id/migration.sql
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions packages/db/prisma/migrations/20250120230752_relate_repo_to_org/migration.sql
View file
Open in desktop

This file was deleted.

Loading