fix(setup,compose): bundle Redis, fix socket reconnect, and harden the setup wizard

apps/sim/lib/core/security/csp-socket-fallback.test.ts

@@ -0,0 +1,46 @@
+/**
+ * @vitest-environment node
+ *
+ * The bundled docker-compose stack runs NODE_ENV=production, serves the app from
+ * localhost, and leaves NEXT_PUBLIC_SOCKET_URL unset. getSocketUrl() falls back
+ * to localhost:3002 for a localhost page regardless of NODE_ENV, so the CSP has
+ * to permit that origin or the browser blocks the handshake and Socket.IO
+ * retries forever. Its own file because vi.mock is hoisted per-module and the
+ * sibling suite needs NEXT_PUBLIC_SOCKET_URL set.
+ */
+import { createEnvMock } from '@sim/testing'
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/env', () =>
+ createEnvMock({
+ NEXT_PUBLIC_APP_URL: 'http://localhost:3000',
+ NEXT_PUBLIC_SOCKET_URL: undefined,
+ })
+)
+
+vi.mock('@/lib/core/config/env-flags', () => ({
+ isDev: false,
+ isHosted: false,
+ isReactGrabEnabled: false,
+}))
+
+import { generateRuntimeCSP } from './csp'
+
+describe('generateRuntimeCSP — socket fallback on a localhost origin', () => {
+ it('permits the default socket origin when NEXT_PUBLIC_SOCKET_URL is unset', () => {
+ const csp = generateRuntimeCSP()
+
+ expect(csp).toContain('http://localhost:3002')
+ expect(csp).toContain('ws://localhost:3002')
+ })
+
+ it('keeps the socket sources inside connect-src', () => {
+ const connectSrc = generateRuntimeCSP()
+ .split(';')
+ .map((directive) => directive.trim())
+ .find((directive) => directive.startsWith('connect-src'))
+
+ expect(connectSrc).toBeDefined()
+ expect(connectSrc).toContain('ws://localhost:3002')
+ })
+})

apps/sim/lib/core/security/csp.ts

@@ -16,6 +16,23 @@ function toWebSocketUrl(httpUrl: string): string {
  return httpUrl.replace('http://', 'ws://').replace('https://', 'wss://')
 }
 
+/**
+ * Kept in sync with LOCALHOST_HOSTNAMES in ../utils/urls by hand: this module is
+ * loaded by next.config.ts before `@/` aliases resolve, so it cannot import from
+ * there (see the note above).
+ */
+const LOCALHOST_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]', '::1'])
+
+/** Mirrors getSocketUrl's localhost check — those origins fall back to DEFAULT_SOCKET_URL. */
+function isLocalhostUrl(url: string): boolean {
+ if (!url) return false
+ try {
+ return LOCALHOST_HOSTNAMES.has(new URL(url).hostname)
+ } catch {
+ return false
+ }
+}
+
 function getHostnameFromUrl(url: string | undefined): string[] {
  if (!url) return []
  try {
@@ -208,7 +225,15 @@ export function buildCSPString(directives: CSPDirectives): string {
 export function generateRuntimeCSP(): string {
  const appUrl = getEnv('NEXT_PUBLIC_APP_URL') || ''
 
- const socketUrl = getEnv('NEXT_PUBLIC_SOCKET_URL') || (isDev ? DEFAULT_SOCKET_URL : '')
+ // Must permit whatever getSocketUrl() actually connects to, or the browser
+ // blocks the handshake and Socket.IO retries forever. That helper falls back
+ // to DEFAULT_SOCKET_URL whenever the page is served from localhost — which
+ // includes a production build (docker compose sets NODE_ENV=production), so
+ // keying this on isDev alone left the bundled stack with a CSP that forbade
+ // its own realtime port. A non-localhost origin still resolves to the page
+ // origin, which appUrl already covers, so nothing is loosened there.
+ const socketUrl =
+ getEnv('NEXT_PUBLIC_SOCKET_URL') || (isDev || isLocalhostUrl(appUrl) ? DEFAULT_SOCKET_URL : '')
  const socketWsUrl = socketUrl ? toWebSocketUrl(socketUrl) : ''
  const ollamaUrl = getEnv('OLLAMA_URL') || (isDev ? DEFAULT_OLLAMA_URL : '')
 

docker-compose.local.yml

@@ -21,15 +21,17 @@ services:
  - ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-at-least-32-chars}
  - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
  - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
- - REDIS_URL=${REDIS_URL:-}
- - COPILOT_API_KEY=${COPILOT_API_KEY}
- - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
+ - REDIS_URL=${REDIS_URL:-redis://redis:6379}
+ - COPILOT_API_KEY=${COPILOT_API_KEY:-}
+ - SIM_AGENT_API_URL=${SIM_AGENT_API_URL:-}
  - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
  - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002}
  - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-}
  depends_on:
  db:
  condition: service_healthy
+ redis:
+ condition: service_healthy
  migrations:
  condition: service_completed_successfully
  realtime:
@@ -55,10 +57,12 @@ services:
  - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
  - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-dev-secret-at-least-32-characters-long}
  - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
- - REDIS_URL=${REDIS_URL:-}
+ - REDIS_URL=${REDIS_URL:-redis://redis:6379}
  depends_on:
  db:
  condition: service_healthy
+ redis:
+ condition: service_healthy
  restart: unless-stopped
  ports:
  - '3002:3002'
@@ -86,6 +90,20 @@ services:
  command: ['bun', 'run', 'db:migrate']
  restart: 'no'
 
+ # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+ # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+ # pub/sub channels have no fallback — without Redis, live status never streams.
+ # Not published to the host: only the app and realtime containers need it, and
+ # binding 6379 would collide with a Redis already running locally.
+ redis:
+ image: redis:7-alpine
+ restart: always
+ healthcheck:
+ test: ['CMD', 'redis-cli', 'ping']
+ interval: 5s
+ timeout: 5s
+ retries: 5
+
  db:
  image: pgvector/pgvector:pg17
  restart: always

docker-compose.prod.yml

@@ -30,20 +30,23 @@ services:
  - ENCRYPTION_KEY=${ENCRYPTION_KEY}
  - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
  - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
- - REDIS_URL=${REDIS_URL:-}
- - COPILOT_API_KEY=${COPILOT_API_KEY}
- - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
+ - REDIS_URL=${REDIS_URL:-redis://redis:6379}
+ - COPILOT_API_KEY=${COPILOT_API_KEY:-}
+ - SIM_AGENT_API_URL=${SIM_AGENT_API_URL:-}
  - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
  - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002}
- # NEXT_PUBLIC_SOCKET_URL is read by the browser. Leaving it unset lets the
- # client default to the page's own origin (assumes the reverse proxy routes
- # /socket.io). Set it explicitly only when the realtime service is on a
- # different host:port from the app (e.g. wss://socket.example.com).
+ # NEXT_PUBLIC_SOCKET_URL is read by the browser. Leave it unset for this
+ # stack: the client already falls back to localhost:3002 for a localhost
+ # page, and a proxied deployment needs the page-origin fallback that an
+ # explicit value would suppress. Set it only when realtime is on a
+ # different host:port (e.g. wss://socket.example.com).
  - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-}
  - ADMISSION_GATE_MAX_INFLIGHT=${ADMISSION_GATE_MAX_INFLIGHT:-500}
  depends_on:
  db:
  condition: service_healthy
+ redis:
+ condition: service_healthy
  migrations:
  condition: service_completed_successfully
  realtime:
@@ -74,10 +77,12 @@ services:
  - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
  - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET}
  - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
- - REDIS_URL=${REDIS_URL:-}
+ - REDIS_URL=${REDIS_URL:-redis://redis:6379}
  depends_on:
  db:
  condition: service_healthy
+ redis:
+ condition: service_healthy
  healthcheck:
  test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:3002/health']
  interval: 90s
@@ -96,6 +101,20 @@ services:
  command: ['bun', 'run', 'db:migrate']
  restart: 'no'
 
+ # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+ # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+ # pub/sub channels have no fallback — without Redis, live status never streams.
+ # Not published to the host: only the app and realtime containers need it, and
+ # binding 6379 would collide with a Redis already running locally.
+ redis:
+ image: redis:7-alpine
+ restart: unless-stopped
+ healthcheck:
+ test: ['CMD', 'redis-cli', 'ping']
+ interval: 5s
+ timeout: 5s
+ retries: 5
+
  db:
  image: pgvector/pgvector:pg17
  restart: unless-stopped

scripts/setup/db.ts

@@ -9,6 +9,18 @@ import { glyph, theme } from './theme.ts'
 
 const DEFAULT_DSN = 'postgresql://postgres:postgres@localhost:5432/simstudio'
 
+/** Postgres' wire message when the password is wrong — a live server, not a dead one. */
+const AUTH_FAILURE = /password authentication failed/i
+
+/**
+ * Percent-encodes the password so characters that are structural in a URL
+ * (`@`, `:`, `/`, `#`, `?`) can't re-parse the DSN into a different host — which
+ * would fail a password that is actually correct.
+ */
+function buildDsn(password: string, hostPort: string | number): string {
+ return `postgresql://postgres:${encodeURIComponent(password)}@localhost:${hostPort}/simstudio`
+}
+
 export function docker(args: string[]): void {
  const result = spawnSync('docker', args, { encoding: 'utf8' })
  if (result.status !== 0) {
@@ -63,7 +75,9 @@ function inspectManagedContainer(): ManagedContainer | null {
 
  return {
  running: running === 'true',
- dsn: `postgresql://postgres:${password}@localhost:${hostPort}/simstudio`,
+ // Read back from the container env verbatim, so it may be a password the
+ // user supplied for an existing volume — encode it like any other.
+ dsn: buildDsn(password, hostPort),
  }
 }
 
@@ -122,6 +136,81 @@ async function promptExternalDsn(): Promise<string> {
  }
 }
 
+const DB_VOLUME = 'sim-postgres-data'
+
+/** True once initdb has run in the volume — PG_VERSION only exists after bootstrap. */
+function volumeInitialized(): boolean {
+ if (spawnSync('docker', ['volume', 'inspect', DB_VOLUME], { stdio: 'ignore' }).status !== 0) {
+ return false
+ }
+ // Read the marker from inside the volume; the image is already local, so this
+ // costs nothing extra and beats assuming "volume exists" means "bootstrapped"
+ // (a failed first run leaves an empty volume behind).
+ return (
+ spawnSync(
+ 'docker',
+ [
+ 'run',
+ '--rm',
+ '-v',
+ `${DB_VOLUME}:/pgdata`,
+ '--entrypoint',
+ 'test',
+ 'pgvector/pgvector:pg17',
+ '-f',
+ '/pgdata/PG_VERSION',
+ ],
+ { stdio: 'ignore' }
+ ).status === 0
+ )
+}
+
+/**
+ * The volume already holds a cluster whose password we cannot read back. Either
+ * the user supplies it, or the data goes — silently generating a new password
+ * would produce a container that never authenticates.
+ */
+async function resolveExistingVolume(): Promise<string> {
+ p.log.warn(
+ `The ${DB_VOLUME} volume already contains a database, but its password is not recoverable — Postgres ignores POSTGRES_PASSWORD on an existing data directory.`
+ )
+ const choice = await p.select({
+ message: 'How should the wizard proceed?',
+ options: [
+ {
+ value: 'password',
+ label: 'Keep the data — I have its password',
+ hint: 'from a previous .env, or your notes',
+ },
+ {
+ value: 'wipe',
+ label: 'Delete the old data and start fresh',
+ hint: `removes the ${DB_VOLUME} volume — this cannot be undone`,
+ },
+ ],
+ initialValue: 'password',
+ })
+ if (choice === 'password') {
+ return p.password({
+ message: `Password for the existing ${DB_VOLUME} database`,
+ validate: (value) => (value ? undefined : 'required'),
+ })
+ }
+ const sure = await p.confirm({
+ message: theme.error(`Permanently delete the ${DB_VOLUME} volume and all its data?`),
+ initialValue: false,
+ })
+ if (!sure) {
+ throw new SetupError('kept the existing database volume, so setup cannot continue.', [
+ 're-run and supply the password, or remove it yourself:',
+ theme.command(`docker volume rm ${DB_VOLUME}`),
+ ])
+ }
+ docker(['volume', 'rm', DB_VOLUME])
+ p.log.step(`Removed ${DB_VOLUME}`)
+ return generateSecret().slice(0, 24)
+}
+
 /**
  * Provisions the managed container, reconciling with one that already exists
  * rather than colliding on the name. Recreating is always an explicit choice —
@@ -142,15 +231,27 @@ async function startManagedContainer(detection: Detection): Promise<string> {
  throw new SetupError(`the existing ${DB_CONTAINER} container is not usable.`, [
  `inspect: ${theme.command(`docker logs ${DB_CONTAINER}`)}`,
  `remove it: ${theme.command(`docker rm -f ${DB_CONTAINER}`)}`,
- `start clean: ${theme.command('docker volume rm sim-postgres-data')} drops its data too`,
+ `start clean: ${theme.command(`docker volume rm ${DB_VOLUME}`)} drops its data too`,
  ])
  }
  docker(['rm', '-f', DB_CONTAINER])
  }
 
- const password = generateSecret().slice(0, 24)
  const hostPort = detection.postgresPortOpen ? 5433 : 5432
- const dsn = `postgresql://postgres:${password}@localhost:${hostPort}/simstudio`
+ // POSTGRES_PASSWORD only applies when initdb runs on an empty data directory.
+ // The volume outlives the container (sim down keeps it, so does `docker rm`),
+ // so once the container is gone the password it was created with is
+ // unrecoverable — inspectManagedContainer reads it from the container, not the
+ // volume. Running with a freshly generated password against an initialized
+ // volume starts a healthy Postgres that rejects every connection with
+ // "password authentication failed", which surfaces as a bogus "container did
+ // not become healthy". Ask instead of guessing.
+ const password = volumeInitialized()
+ ? await resolveExistingVolume()
+ : generateSecret().slice(0, 24)
+ // A user-supplied password can contain @ : / # — raw interpolation would
+ // re-parse the DSN into a different host and fail a password that is correct.
+ const dsn = buildDsn(password, hostPort)
  docker([
  'run',
  '-d',
@@ -159,7 +260,7 @@ async function startManagedContainer(detection: Detection): Promise<string> {
  '--label',
  'managed-by=sim-setup',
  '-v',
-'sim-postgres-data:/var/lib/postgresql/data',
+`${DB_VOLUME}:/var/lib/postgresql/data`,
  '-e',
  `POSTGRES_PASSWORD=${password}`,
  '-e',
@@ -170,9 +271,31 @@ async function startManagedContainer(detection: Detection): Promise<string> {
  ])
  const spin = p.spinner()
  spin.start(`Starting ${DB_CONTAINER} container on :${hostPort}…`)
- const healthy = await waitFor(async () => (await pgProbe(dsn)).ok, 45_000, 1500)
+ let lastError = ''
+ const healthy = await waitFor(
+ async () => {
+ const probe = await pgProbe(dsn)
+ if (!probe.ok) lastError = probe.error ?? ''
+ return probe.ok
+ },
+ 45_000,
+ 1500
+ )
  if (!healthy) {
  spin.stop(`${glyph.fail} container did not become healthy`)
+ // Postgres running and refusing the password is a different failure from
+ // Postgres never starting, and it is the likely one on the keep-the-volume
+ // path. Reporting it as "did not become healthy" is the exact confusion
+ // this whole change set exists to remove.
+ if (AUTH_FAILURE.test(lastError)) {
+ throw new SetupError(
+ `Postgres started, but rejected that password for the existing ${DB_VOLUME} volume.`,
+ [
+ 're-run and enter the password the volume was created with',
+ `or discard the old data: ${theme.command(`docker rm -f ${DB_CONTAINER} && docker volume rm ${DB_VOLUME}`)}`,
+ ]
+ )
+ }
  const logs = spawnSync('docker', ['logs', '--tail', '20', DB_CONTAINER], { encoding: 'utf8' })
  throw new SetupError(
  `the Postgres container failed to start. Last logs:\n${logs.stdout}${logs.stderr}`,