From 115f1daf5bafccb0008830a23827458dfd6cb74c Mon Sep 17 00:00:00 2001 From: Mikhail Vlasenko Date: Wed, 12 Aug 2026 10:58:54 +1100 Subject: [PATCH 1/3] fix(dev): make local service host ports configurable Signed-off-by: Mikhail Vlasenko --- .env.example | 31 ++++- CONTRIBUTING.md | 8 +- Justfile | 16 +-- TESTING.md | 6 +- docker-compose.yml | 14 +- scripts/dev-setup.sh | 26 ++-- scripts/lib/dev-service-env.sh | 89 +++++++++++++ scripts/start-relay-for-tests.sh | 31 ++--- scripts/test-dev-service-ports.sh | 209 ++++++++++++++++++++++++++++++ 9 files changed, 382 insertions(+), 48 deletions(-) create mode 100644 scripts/lib/dev-service-env.sh create mode 100755 scripts/test-dev-service-ports.sh diff --git a/.env.example b/.env.example index 0f7bbba6f13..a94649d18be 100644 --- a/.env.example +++ b/.env.example @@ -7,17 +7,24 @@ # All defaults here work with `docker compose up` out of the box. # # Service ports (defaults): -# Postgres → localhost:5432 -# Redis → localhost:6379 -# Typesense → localhost:8108 -# Adminer → localhost:8082 (DB browser UI) +# Postgres → localhost:5432 +# Redis → localhost:6379 +# Typesense → localhost:8108 +# Adminer → localhost:8082 (DB browser UI) +# Keycloak → localhost:8180 (OAuth/OIDC testing) +# MinIO API → localhost:9000 +# MinIO console → localhost:9001 +# Prometheus → localhost:9090 # -# Note: If port 8082 conflicts, change the adminer port in docker-compose.yml +# Change host ports here; Docker Compose keeps the service ports inside the +# dev network unchanged. # ============================================================================= # ----------------------------------------------------------------------------- # Database (Postgres 17) # ----------------------------------------------------------------------------- +# Keep DATABASE_URL aligned with PGHOST and PGPORT. +# Compose uses PGPORT as the published host port. DATABASE_URL=postgres://buzz:buzz_dev@localhost:5432/buzz # Optional read-replica URL; unset/blank keeps all reads on the writer. # READ_DATABASE_URL=postgres://buzz:buzz_dev@localhost:5433/buzz @@ -31,6 +38,8 @@ PGDATABASE=buzz # Redis 7 # ----------------------------------------------------------------------------- REDIS_URL=redis://localhost:6379 +# Host port published by Docker Compose. Keep it aligned with REDIS_URL. +REDIS_PORT=6379 # Max connections in the relay's shared Redis pool (default 16). # BUZZ_REDIS_POOL_SIZE=16 @@ -44,6 +53,15 @@ REDIS_URL=redis://localhost:6379 TYPESENSE_API_KEY=buzz_dev_key TYPESENSE_URL=http://localhost:8108 +# ----------------------------------------------------------------------------- +# Auxiliary local service ports +# ----------------------------------------------------------------------------- +ADMINER_PORT=8082 +KEYCLOAK_PORT=8180 +MINIO_API_PORT=9000 +MINIO_CONSOLE_PORT=9001 +PROMETHEUS_PORT=9090 + # ----------------------------------------------------------------------------- # Relay (WebSocket server) # ----------------------------------------------------------------------------- @@ -85,7 +103,8 @@ RELAY_URL=ws://localhost:3000 # ----------------------------------------------------------------------------- # S3-Compatible Object Storage (media + Git/CAS) # ----------------------------------------------------------------------------- -# The local MinIO container is reachable from host processes at localhost:9000. +# The local MinIO container is reachable from host processes at the port in +# MINIO_API_PORT. Keep BUZZ_S3_ENDPOINT aligned if that port changes. # Path style keeps the bucket in the URL path and is required by this local DNS # setup. Use `virtual` only when the provider requires bucket-as-subdomain URLs. BUZZ_S3_ENDPOINT=http://localhost:9000 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db0aea637fe..a01afe68c63 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -154,8 +154,12 @@ it is safe to re-run. `just setup` then starts Docker services (Postgres on `:5432`, Redis on `:6379`, Adminer on `:8082`, Keycloak on `:8180` for local OAuth/OIDC testing, MinIO on -`:9000` for media storage, and Prometheus on `:9090` for metrics) and runs all -pending database migrations. +`:9000`/`:9001` for media storage and its console, and Prometheus on `:9090` for +metrics) and runs all pending database migrations. These are host-port defaults; +override `PGPORT`, `REDIS_PORT`, `ADMINER_PORT`, `KEYCLOAK_PORT`, +`MINIO_API_PORT`, `MINIO_CONSOLE_PORT`, or `PROMETHEUS_PORT` in `.env` when a +port is already occupied. Keep `DATABASE_URL`, `REDIS_URL`, and +`BUZZ_S3_ENDPOINT` aligned with their corresponding host-port overrides. ### Running the Relay and Desktop App diff --git a/Justfile b/Justfile index 5b2ed88c952..64b54543995 100644 --- a/Justfile +++ b/Justfile @@ -167,14 +167,8 @@ _ensure-sidecar-stubs: _ensure-services: #!/usr/bin/env bash set -euo pipefail - pg=$(docker inspect --format '{{"{{"}}.State.Health.Status{{"}}"}}' buzz-postgres 2>/dev/null || echo "not_found") - redis=$(docker inspect --format '{{"{{"}}.State.Health.Status{{"}}"}}' buzz-redis 2>/dev/null || echo "not_found") - if [[ "$pg" == "healthy" && "$redis" == "healthy" ]]; then - echo "Services already healthy" - exit 0 - fi - echo "Starting services..." - docker compose up -d || true + echo "Starting or reconciling services..." + docker compose up -d echo -n "Waiting for services" for i in $(seq 1 40); do pg=$(docker inspect --format '{{"{{"}}.State.Health.Status{{"}}"}}' buzz-postgres 2>/dev/null || echo "not_found") @@ -286,10 +280,14 @@ desktop-e2e-pre-push: _ensure-migrations cd {{desktop_dir}} && pnpm build:e2e && pnpm exec playwright test --only-changed=origin/main # Run all checks suitable for CI / pre-push (no infra needed) -ci: check test-unit desktop-test desktop-build desktop-tauri-check desktop-tauri-test web-build mobile-test +ci: check test-dev-service-ports test-unit desktop-test desktop-build desktop-tauri-check desktop-tauri-test web-build mobile-test # ─── Test ───────────────────────────────────────────────────────────────────── +# Verify default/custom host-port mappings and service reconciliation behavior +test-dev-service-ports: + ./scripts/test-dev-service-ports.sh + # Run all tests (unit + integration) test: ./scripts/run-tests.sh all diff --git a/TESTING.md b/TESTING.md index 29d07a80de0..90ac5455ecb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -35,7 +35,8 @@ just setup # start Docker services, run migrations > **Already running Buzz Desktop?** Desktop uses the same Docker container > names (`buzz-postgres`, `buzz-redis`) and the same -> default ports (`:5432`, `:6379`). `just setup` will reuse those +> default ports (`:5432`, `:6379`). Host ports can be overridden in `.env`; +> keep `DATABASE_URL`/`PGPORT` and `REDIS_URL`/`REDIS_PORT` aligned. `just setup` will reuse those > services, so **your test relay writes into Desktop's database**. That's > fine for read/write smoke tests, but: `just reset` wipes Desktop's data > along with yours. If you need isolation, stop Desktop first or run the @@ -314,7 +315,10 @@ out of the box with `just setup` or `just relay`. Common overrides: | `BUZZ_METRICS_PORT` | `9102` | Prometheus `/metrics` | | `RELAY_URL` | `ws://localhost:3000` | Advertised in NIP-11 / NIP-42 challenges. **Note: no `BUZZ_` prefix.** | | `DATABASE_URL` | `postgres://buzz:buzz_dev@localhost:5432/buzz` | | +| `PGPORT` | `5432` | Postgres host port published by the root dev Compose stack; keep aligned with `DATABASE_URL` | | `REDIS_URL` | `redis://localhost:6379` | | +| `REDIS_PORT` | `6379` | Redis host port published by the root dev Compose stack; keep aligned with `REDIS_URL` | +| `MINIO_API_PORT` | `9000` | MinIO API host port; keep aligned with `BUZZ_S3_ENDPOINT` | | `BUZZ_REQUIRE_AUTH_TOKEN` | `false` | When true, REST requires NIP-98 (no `X-Pubkey` fallback) | | `BUZZ_REQUIRE_RELAY_MEMBERSHIP` | `false` | When true, only pubkeys in `relay_members` can connect | | `BUZZ_DRAIN_JITTER_MS` | `0` (off) | Per-connection upper bound, in ms, for the random delay before each live WebSocket gets its `1012 Service Restart` close on graceful shutdown. `0` closes every socket at once (the previous behavior). A positive value spreads closes uniformly over `[1, value]` ms to avoid a reconnect thundering herd on rolling deploys. Values above `20000` are capped to `20000` (`MAX_DRAIN_JITTER_MS`) to leave close-frame delivery headroom under the relay's 30s hard-drain timeout. Empty or whitespace-only is treated as unset (off); a non-integer fails startup loudly. | diff --git a/docker-compose.yml b/docker-compose.yml index e7dc09fafc4..c3ef5f666bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: POSTGRES_DB: buzz PGDATA: /var/lib/postgresql/data ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:${PGPORT:-5432}:5432" volumes: - postgres-data:/var/lib/postgresql/data networks: @@ -34,7 +34,7 @@ services: image: redis:7-alpine container_name: buzz-redis ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:${REDIS_PORT:-6379}:6379" networks: - buzz-net healthcheck: @@ -56,7 +56,7 @@ services: image: adminer:latest container_name: buzz-adminer ports: - - "127.0.0.1:8082:8080" + - "127.0.0.1:${ADMINER_PORT:-8082}:8080" networks: - buzz-net depends_on: @@ -82,7 +82,7 @@ services: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin ports: - - "127.0.0.1:8180:8080" + - "127.0.0.1:${KEYCLOAK_PORT:-8180}:8080" networks: - buzz-net healthcheck: @@ -108,8 +108,8 @@ services: MINIO_ROOT_USER: buzz_dev MINIO_ROOT_PASSWORD: buzz_dev_secret ports: - - "127.0.0.1:9000:9000" - - "127.0.0.1:9001:9001" + - "127.0.0.1:${MINIO_API_PORT:-9000}:9000" + - "127.0.0.1:${MINIO_CONSOLE_PORT:-9001}:9001" volumes: - minio-data:/data networks: @@ -152,7 +152,7 @@ services: image: prom/prometheus:latest container_name: buzz-prometheus ports: - - "127.0.0.1:9090:9090" + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index ae358517b34..be387113e80 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -12,6 +12,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev-service-env.sh" + # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -67,6 +70,15 @@ load_env() { export PGPASSWORD="${PGPASSWORD:-buzz_dev}" export PGDATABASE="${PGDATABASE:-buzz}" export REDIS_URL="${REDIS_URL:-redis://localhost:6379}" + export REDIS_PORT="${REDIS_PORT:-6379}" + export MINIO_API_PORT="${MINIO_API_PORT:-9000}" + export BUZZ_S3_ENDPOINT="${BUZZ_S3_ENDPOINT:-http://localhost:9000}" +} + +validate_env() { + validate_local_service_port "DATABASE_URL" "${DATABASE_URL}" "${PGPORT}" "5432" + validate_local_service_port "REDIS_URL" "${REDIS_URL}" "${REDIS_PORT}" "6379" + validate_local_service_port "BUZZ_S3_ENDPOINT" "${BUZZ_S3_ENDPOINT}" "${MINIO_API_PORT}" "80" } cleanup_legacy_sprout_containers() { @@ -76,7 +88,7 @@ cleanup_legacy_sprout_containers() { return fi - warn "Stopping/removing legacy sprout-* dev containers so buzz-* containers can bind the standard ports" + warn "Stopping/removing legacy sprout-* dev containers so buzz-* containers can bind the configured ports" echo "${legacy_containers}" | xargs docker stop >/dev/null 2>&1 || true echo "${legacy_containers}" | xargs docker rm >/dev/null 2>&1 || true success "Legacy sprout-* containers removed (volumes preserved)" @@ -86,14 +98,11 @@ fail_if_local_redis_blocks_compose() { if ! command -v lsof >/dev/null 2>&1; then return fi - if docker ps --format '{{.Names}}' | grep -qx 'buzz-redis'; then - return - fi local redis_pids - redis_pids=$(lsof -nP -iTCP:6379 -sTCP:LISTEN 2>/dev/null | awk 'NR > 1 && $1 == "redis-ser" {print $2}' | sort -u | tr ' + redis_pids=$(lsof -nP -iTCP:"${REDIS_PORT}" -sTCP:LISTEN 2>/dev/null | awk 'NR > 1 && $1 == "redis-ser" {print $2}' | sort -u | tr ' ' ' ' || true) if [[ -n "${redis_pids}" ]]; then - error "Local Redis is already listening on port 6379 (pid(s): ${redis_pids}). Stop it before running setup: brew services stop redis" + error "Local Redis is already listening on port ${REDIS_PORT} (pid(s): ${redis_pids}). Stop it before running setup: brew services stop redis" exit 1 fi } @@ -105,6 +114,7 @@ postgres_accepting_connections() { } load_env +validate_env cleanup_legacy_sprout_containers fail_if_local_redis_blocks_compose @@ -187,8 +197,8 @@ echo -e "${GREEN}=======================================================${NC}" echo "" echo -e " ${BLUE}Postgres${NC} ${DATABASE_URL}" echo -e " ${BLUE}Redis${NC} ${REDIS_URL}" -echo -e " ${BLUE}Adminer${NC} http://localhost:8082 (DB browser)" -echo -e " ${BLUE}Keycloak${NC} http://localhost:8180 (admin / admin — local OAuth testing)" +echo -e " ${BLUE}Adminer${NC} http://localhost:${ADMINER_PORT:-8082} (DB browser)" +echo -e " ${BLUE}Keycloak${NC} http://localhost:${KEYCLOAK_PORT:-8180} (admin / admin — local OAuth testing)" echo "" echo -e " ${YELLOW}Next steps:${NC}" echo -e " just relay # start the relay (terminal 1)" diff --git a/scripts/lib/dev-service-env.sh b/scripts/lib/dev-service-env.sh new file mode 100644 index 00000000000..8c7e9738410 --- /dev/null +++ b/scripts/lib/dev-service-env.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Shared validation helpers for root development service configuration. + +validate_local_service_port() { + local service="$1" + local url="$2" + local expected_port="$3" + local default_url_port="$4" + local parsed + + # shellcheck disable=SC2016 + if ! parsed=$(node -e ' + try { + const url = new URL(process.argv[1]); + const host = url.hostname.toLowerCase(); + const isLocal = host === "localhost" || host === "127.0.0.1" || host === "[::1]"; + const port = url.port || process.argv[2]; + process.stdout.write(`${isLocal ? "local" : "remote"}\t${port}`); + } catch { + process.exit(1); + } + ' "${url}" "${default_url_port}"); then + echo "${service} URL is not a valid absolute URL" >&2 + return 1 + fi + + local location="${parsed%%$'\t'*}" + local actual_port="${parsed#*$'\t'}" + if [[ "${location}" == "local" && "${actual_port}" != "${expected_port}" ]]; then + echo "${service} URL uses local port ${actual_port}, but its Compose host port is ${expected_port}" >&2 + return 1 + fi +} + +compose_service_host_port() { + local service="$1" + local container_port="$2" + local binding + local host_port + + if ! binding=$(docker compose port "${service}" "${container_port}" 2>/dev/null); then + echo "Could not resolve the published port for ${service}:${container_port}" >&2 + return 1 + fi + binding="${binding##*$'\n'}" + host_port="${binding##*:}" + if [[ ! "${host_port}" =~ ^[0-9]+$ ]]; then + echo "Invalid published port for ${service}:${container_port}" >&2 + return 1 + fi + + printf '%s\n' "${host_port}" +} + +# Configure the shared relay-test launcher from the ports Compose actually +# published. Service hosts and credentials stay pinned to the local dev stack, +# so a developer's remote DATABASE_URL/PG* environment can never become an +# auto-approved schema target. +configure_local_compose_service_env() { + local postgres_port + local redis_port + local minio_port + + postgres_port=$(compose_service_host_port postgres 5432) + redis_port=$(compose_service_host_port redis 6379) + minio_port=$(compose_service_host_port minio 9000) + + export PGHOST=localhost + export PGPORT="${postgres_port}" + export PGUSER=buzz + export PGPASSWORD=buzz_dev + export PGDATABASE=buzz + export DATABASE_URL="postgres://buzz:buzz_dev@localhost:${PGPORT}/buzz" + + export PGSCHEMA_PLAN_HOST=localhost + export PGSCHEMA_PLAN_PORT="${PGPORT}" + export PGSCHEMA_PLAN_DB=buzz + export PGSCHEMA_PLAN_USER=buzz + export PGSCHEMA_PLAN_PASSWORD=buzz_dev + + export REDIS_PORT="${redis_port}" + export REDIS_URL="redis://localhost:${REDIS_PORT}" + + export MINIO_API_PORT="${minio_port}" + export BUZZ_S3_ENDPOINT="http://localhost:${MINIO_API_PORT}" + export BUZZ_S3_ACCESS_KEY=buzz_dev + export BUZZ_S3_SECRET_KEY=buzz_dev_secret + export BUZZ_S3_BUCKET=buzz-media +} diff --git a/scripts/start-relay-for-tests.sh b/scripts/start-relay-for-tests.sh index b9d93935c08..f366661cb69 100755 --- a/scripts/start-relay-for-tests.sh +++ b/scripts/start-relay-for-tests.sh @@ -21,6 +21,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev-service-env.sh" + # ── Defaults ────────────────────────────────────────────────────────────────── CARGO_PROFILE="${CARGO_PROFILE:-ci}" @@ -63,13 +66,18 @@ cd "${REPO_ROOT}" log "Starting docker compose services..." docker compose up -d postgres redis minio minio-init +# Compose reads .env and applies its host-port overrides. Resolve the ports from +# the running local stack rather than sourcing arbitrary service URLs or +# credentials from .env into this auto-approving test launcher. +configure_local_compose_service_env + # ── Wait for services to be healthy ────────────────────────────────────────── wait_healthy() { local service="$1" local container="$2" log "Waiting for ${service}..." - for attempt in $(seq 1 60); do + for _attempt in $(seq 1 60); do status=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "not_found") if [ "${status}" = "healthy" ]; then ok "${service} is healthy" @@ -89,20 +97,9 @@ wait_healthy "MinIO" "buzz-minio" # ── Apply database schema ──────────────────────────────────────────────────── log "Applying database schema..." -export PGHOST=localhost -export PGPORT=5432 -export PGUSER=buzz -export PGPASSWORD=buzz_dev -export PGDATABASE=buzz # Use the already-running docker postgres for desired-state planning instead of # downloading an embedded Postgres from Maven Central (transient-fetch flake source). -export PGSCHEMA_PLAN_HOST=localhost -export PGSCHEMA_PLAN_PORT=5432 -export PGSCHEMA_PLAN_DB=buzz -export PGSCHEMA_PLAN_USER=buzz -export PGSCHEMA_PLAN_PASSWORD=buzz_dev - ./bin/pgschema apply --file schema/schema.sql --auto-approve docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \ psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql @@ -166,8 +163,12 @@ if [[ "${BUZZ_REQUIRE_RELAY_MEMBERSHIP:-}" == "true" ]]; then fi nohup env \ - DATABASE_URL=postgres://buzz:buzz_dev@localhost:5432/buzz \ - REDIS_URL=redis://localhost:6379 \ + DATABASE_URL="${DATABASE_URL}" \ + REDIS_URL="${REDIS_URL}" \ + BUZZ_S3_ENDPOINT="${BUZZ_S3_ENDPOINT}" \ + BUZZ_S3_ACCESS_KEY="${BUZZ_S3_ACCESS_KEY}" \ + BUZZ_S3_SECRET_KEY="${BUZZ_S3_SECRET_KEY}" \ + BUZZ_S3_BUCKET="${BUZZ_S3_BUCKET}" \ RELAY_URL=ws://localhost:3000 \ BUZZ_BIND_ADDR=0.0.0.0:3000 \ BUZZ_REQUIRE_AUTH_TOKEN=false \ @@ -180,7 +181,7 @@ echo $! > /tmp/buzz-relay.pid # ── Poll readiness ─────────────────────────────────────────────────────────── log "Waiting for relay readiness..." -for attempt in $(seq 1 60); do +for _attempt in $(seq 1 60); do if ! kill -0 "$(cat /tmp/buzz-relay.pid)" 2>/dev/null; then err "Relay process died" cat /tmp/buzz-relay.log diff --git a/scripts/test-dev-service-ports.sh b/scripts/test-dev-service-ports.sh new file mode 100755 index 00000000000..7d3a8f79e0a --- /dev/null +++ b/scripts/test-dev-service-ports.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# Regression checks for configurable host ports in the root dev Compose stack. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev-service-env.sh" + +assert_eq() { + local actual="$1" + local expected="$2" + local description="$3" + if [[ "${actual}" != "${expected}" ]]; then + echo "${description}: expected ${expected}, got ${actual}" >&2 + exit 1 + fi +} + +validate_local_service_port "Postgres" "postgres://buzz:secret@localhost:15432/buzz" "15432" "5432" +validate_local_service_port "Redis" "redis://127.0.0.1:16379" "16379" "6379" +validate_local_service_port "MinIO" "http://[::1]:19000" "19000" "80" +validate_local_service_port "External Postgres" "postgres://db.example.com:5432/buzz" "15432" "5432" + +validation_error="${TMP_DIR}/validation-error" +if validate_local_service_port \ + "Postgres" \ + "postgres://buzz:do-not-print@localhost:5432/buzz" \ + "15432" \ + "5432" 2> "${validation_error}"; then + echo "mismatched local URL and host port unexpectedly passed validation" >&2 + exit 1 +fi +if grep -q 'do-not-print' "${validation_error}"; then + echo "port validation leaked URL credentials" >&2 + exit 1 +fi + +# The shared relay launcher may honor Compose host-port overrides, but it must +# never apply schemas to or start against service URLs and credentials inherited +# from a developer's environment. +( + # shellcheck disable=SC2329 + docker() { + case "$*" in + "compose port postgres 5432") echo "127.0.0.1:15432" ;; + "compose port redis 6379") echo "127.0.0.1:16379" ;; + "compose port minio 9000") echo "127.0.0.1:19000" ;; + *) echo "unexpected docker invocation: $*" >&2; return 1 ;; + esac + } + + export DATABASE_URL="postgres://remote:secret@db.example.com:5432/production" + export PGHOST="db.example.com" + export PGPORT="5432" + export PGUSER="remote" + export PGPASSWORD="secret" + export PGDATABASE="production" + export PGSCHEMA_PLAN_HOST="db.example.com" + export PGSCHEMA_PLAN_PORT="5432" + export PGSCHEMA_PLAN_DB="production" + export PGSCHEMA_PLAN_USER="remote" + export PGSCHEMA_PLAN_PASSWORD="secret" + export REDIS_URL="redis://cache.example.com:6379" + export BUZZ_S3_ENDPOINT="https://storage.example.com" + export BUZZ_S3_ACCESS_KEY="remote-key" + export BUZZ_S3_SECRET_KEY="remote-secret" + export BUZZ_S3_BUCKET="production" + + configure_local_compose_service_env + + assert_eq "${DATABASE_URL}" "postgres://buzz:buzz_dev@localhost:15432/buzz" "test relay database URL" + assert_eq "${PGSCHEMA_PLAN_HOST}" "localhost" "schema plan host" + assert_eq "${PGSCHEMA_PLAN_PORT}" "15432" "schema plan port" + assert_eq "${PGSCHEMA_PLAN_DB}" "buzz" "schema plan database" + assert_eq "${PGSCHEMA_PLAN_USER}" "buzz" "schema plan user" + assert_eq "${PGSCHEMA_PLAN_PASSWORD}" "buzz_dev" "schema plan password" + assert_eq "${REDIS_URL}" "redis://localhost:16379" "test relay Redis URL" + assert_eq "${BUZZ_S3_ENDPOINT}" "http://localhost:19000" "test relay MinIO endpoint" + assert_eq "${BUZZ_S3_ACCESS_KEY}" "buzz_dev" "test relay MinIO access key" + assert_eq "${BUZZ_S3_SECRET_KEY}" "buzz_dev_secret" "test relay MinIO secret key" + assert_eq "${BUZZ_S3_BUCKET}" "buzz-media" "test relay MinIO bucket" +) + +render_config() { + local output="$1" + shift + env \ + -u PGPORT \ + -u REDIS_PORT \ + -u ADMINER_PORT \ + -u KEYCLOAK_PORT \ + -u MINIO_API_PORT \ + -u MINIO_CONSOLE_PORT \ + -u PROMETHEUS_PORT \ + -u PGUSER \ + -u PGPASSWORD \ + -u PGDATABASE \ + -u BUZZ_S3_ACCESS_KEY \ + -u BUZZ_S3_SECRET_KEY \ + -u BUZZ_S3_BUCKET \ + "$@" docker compose \ + --project-directory "${REPO_ROOT}" \ + --env-file /dev/null \ + config --format json > "${output}" +} + +assert_config() { + local config="$1" + shift + # shellcheck disable=SC2016 + node -e ' + const fs = require("node:fs"); + const config = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + const expected = JSON.parse(process.argv[2]); + + for (const [service, ports] of Object.entries(expected)) { + const actual = (config.services[service].ports ?? []).map((port) => ({ + host_ip: port.host_ip, + published: String(port.published), + target: Number(port.target), + })); + if (JSON.stringify(actual) !== JSON.stringify(ports)) { + throw new Error(`${service} ports: expected ${JSON.stringify(ports)}, got ${JSON.stringify(actual)}`); + } + } + + const postgres = config.services.postgres.environment; + if (postgres.POSTGRES_USER !== "buzz" || postgres.POSTGRES_PASSWORD !== "buzz_dev" || postgres.POSTGRES_DB !== "buzz") { + throw new Error("host-port overrides must not change Postgres initialization credentials"); + } + + const minio = config.services.minio.environment; + if (minio.MINIO_ROOT_USER !== "buzz_dev" || minio.MINIO_ROOT_PASSWORD !== "buzz_dev_secret") { + throw new Error("host-port overrides must not change MinIO initialization credentials"); + } + ' "$config" "$1" +} + +default_config="${TMP_DIR}/default.json" +render_config "${default_config}" +assert_config "${default_config}" '{ + "postgres":[{"host_ip":"127.0.0.1","published":"5432","target":5432}], + "redis":[{"host_ip":"127.0.0.1","published":"6379","target":6379}], + "adminer":[{"host_ip":"127.0.0.1","published":"8082","target":8080}], + "keycloak":[{"host_ip":"127.0.0.1","published":"8180","target":8080}], + "minio":[ + {"host_ip":"127.0.0.1","published":"9000","target":9000}, + {"host_ip":"127.0.0.1","published":"9001","target":9001} + ], + "prometheus":[{"host_ip":"127.0.0.1","published":"9090","target":9090}] +}' + +custom_config="${TMP_DIR}/custom.json" +render_config "${custom_config}" \ + PGPORT=15432 \ + REDIS_PORT=16379 \ + ADMINER_PORT=18082 \ + KEYCLOAK_PORT=18180 \ + MINIO_API_PORT=19000 \ + MINIO_CONSOLE_PORT=19001 \ + PROMETHEUS_PORT=19090 \ + PGUSER=must-not-propagate \ + PGPASSWORD=must-not-propagate \ + PGDATABASE=must-not-propagate \ + BUZZ_S3_ACCESS_KEY=must-not-propagate \ + BUZZ_S3_SECRET_KEY=must-not-propagate \ + BUZZ_S3_BUCKET=must-not-propagate +assert_config "${custom_config}" '{ + "postgres":[{"host_ip":"127.0.0.1","published":"15432","target":5432}], + "redis":[{"host_ip":"127.0.0.1","published":"16379","target":6379}], + "adminer":[{"host_ip":"127.0.0.1","published":"18082","target":8080}], + "keycloak":[{"host_ip":"127.0.0.1","published":"18180","target":8080}], + "minio":[ + {"host_ip":"127.0.0.1","published":"19000","target":9000}, + {"host_ip":"127.0.0.1","published":"19001","target":9001} + ], + "prometheus":[{"host_ip":"127.0.0.1","published":"19090","target":9090}] +}' + +docker_calls="${TMP_DIR}/docker-calls" +mock_bin="${TMP_DIR}/bin" +mkdir -p "${mock_bin}" +# shellcheck disable=SC2016 +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + 'if [[ "$1" == "inspect" ]]; then' \ + ' echo healthy' \ + 'elif [[ "$1" == "compose" ]]; then' \ + ' printf "%s\n" "$*" >> "${DOCKER_CALLS}"' \ + 'else' \ + ' echo "unexpected docker invocation: $*" >&2' \ + ' exit 1' \ + 'fi' > "${mock_bin}/docker" +chmod +x "${mock_bin}/docker" + +PATH="${mock_bin}:${PATH}" DOCKER_CALLS="${docker_calls}" \ + "${REPO_ROOT}/bin/just" --justfile "${REPO_ROOT}/Justfile" _ensure-services + +if ! grep -qx 'compose up -d' "${docker_calls}"; then + echo "_ensure-services did not reconcile Compose when existing services were healthy" >&2 + exit 1 +fi + +echo "Dev service port checks passed" From 55aeb80cdf3a8d3ddbef27351954109005ffcf70 Mon Sep 17 00:00:00 2001 From: Mikhail Vlasenko Date: Sat, 15 Aug 2026 11:45:00 +1100 Subject: [PATCH 2/3] fix(dev): make setup independent of PATH Signed-off-by: Mikhail Vlasenko --- TESTING.md | 12 +++--- scripts/dev-setup.sh | 4 ++ scripts/test-dev-service-ports.sh | 67 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/TESTING.md b/TESTING.md index 90ac5455ecb..30c5fbd37e2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -36,11 +36,13 @@ just setup # start Docker services, run migrations > **Already running Buzz Desktop?** Desktop uses the same Docker container > names (`buzz-postgres`, `buzz-redis`) and the same > default ports (`:5432`, `:6379`). Host ports can be overridden in `.env`; -> keep `DATABASE_URL`/`PGPORT` and `REDIS_URL`/`REDIS_PORT` aligned. `just setup` will reuse those -> services, so **your test relay writes into Desktop's database**. That's -> fine for read/write smoke tests, but: `just reset` wipes Desktop's data -> along with yours. If you need isolation, stop Desktop first or run the -> dev stack on a different Compose project +> keep `DATABASE_URL`/`PGPORT` and `REDIS_URL`/`REDIS_PORT` aligned. Changing +> an override reconciles the shared Compose stack, so a running Desktop or +> relay using the old ports will lose its service connections. `just setup` +> will reuse those services, so **your test relay writes into Desktop's +> database**. That's fine for read/write smoke tests, but: `just reset` wipes +> Desktop's data along with yours. If you need isolation, stop Desktop first +> or run the dev stack on a different Compose project > (`COMPOSE_PROJECT_NAME=buzz-dev docker compose …`). `just reset` wipes all local data and starts over — **including Buzz diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index be387113e80..7ee9fdcb30e 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -12,6 +12,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# `just setup` runs `bootstrap` in a separate recipe. Keep this script +# self-contained when it validates URLs with the Hermit-provided node shim. +export PATH="${REPO_ROOT}/bin:${PATH}" + # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/dev-service-env.sh" diff --git a/scripts/test-dev-service-ports.sh b/scripts/test-dev-service-ports.sh index 7d3a8f79e0a..e6a34cbb88e 100755 --- a/scripts/test-dev-service-ports.sh +++ b/scripts/test-dev-service-ports.sh @@ -206,4 +206,71 @@ if ! grep -qx 'compose up -d' "${docker_calls}"; then exit 1 fi +# `just setup` runs bootstrap in a separate recipe, so dev-setup itself must +# add the repository's Hermit shims before validating URLs with node. Exercise +# that boundary without a system node binary or a real Docker daemon. +setup_probe_root="${TMP_DIR}/setup-probe" +mkdir -p "${setup_probe_root}/scripts/lib" "${setup_probe_root}/bin" "${setup_probe_root}/mock-bin" +cp "${REPO_ROOT}/scripts/dev-setup.sh" "${setup_probe_root}/scripts/dev-setup.sh" +cp "${REPO_ROOT}/scripts/lib/dev-service-env.sh" "${setup_probe_root}/scripts/lib/dev-service-env.sh" +cp "${REPO_ROOT}/bin/node" "${setup_probe_root}/bin/node" + +cat > "${setup_probe_root}/mock-hermit" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +# Mimic the generated Hermit node shim closely enough for the URL parser used +# by dev-service-env.sh. The probe only uses the three default local URLs. +url="${7}" +default_port="${8}" +case "${url}" in + postgres://*@localhost:5432/*) printf 'local\t5432' ;; + redis://localhost:6379) printf 'local\t6379' ;; + http://localhost:9000) printf 'local\t9000' ;; + *) printf 'local\t%s' "${default_port}" ;; +esac +EOF +chmod +x "${setup_probe_root}/mock-hermit" + +cat > "${setup_probe_root}/mock-bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +case "$*" in + info) exit 0 ;; + "ps -a --format {{.Names}}") exit 0 ;; + *) echo "unexpected docker invocation: $*" >&2; exit 1 ;; +esac +EOF +chmod +x "${setup_probe_root}/mock-bin/docker" + +cat > "${setup_probe_root}/bin/just" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$1" == "_ensure-services" ]]; then + echo 'dev-setup reached Compose' + exit 42 +fi +echo "unexpected just invocation: $*" >&2 +exit 1 +EOF +chmod +x "${setup_probe_root}/bin/just" + +setup_probe_output="${TMP_DIR}/setup-probe-output" +if ( + cd "${setup_probe_root}" + PATH="${setup_probe_root}/mock-bin:/usr/bin:/bin" \ + HERMIT_EXE="${setup_probe_root}/mock-hermit" \ + ./scripts/dev-setup.sh +) > "${setup_probe_output}" 2>&1; then + echo "dev-setup unexpectedly completed the no-system-node probe" >&2 + exit 1 +fi +if ! grep -Fqx 'dev-setup reached Compose' "${setup_probe_output}"; then + cat "${setup_probe_output}" >&2 + echo "dev-setup did not use its Hermit node shim before validating service URLs" >&2 + exit 1 +fi + echo "Dev service port checks passed" From ac7abeb5bbc55ab92d52cca0c71ab9fc6b0163ee Mon Sep 17 00:00:00 2001 From: Ravneet Arora Date: Mon, 17 Aug 2026 12:51:19 -0400 Subject: [PATCH 3/3] fix(dev): harden local service URL validation Canonicalize loopback aliases before comparing configured service ports, reject unspecified bind addresses, and cover the setup boundary before Compose or migrations can run. Co-authored-by: Ravneet Arora Signed-off-by: Ravneet Arora --- scripts/lib/dev-service-env.sh | 45 +++++++++- scripts/test-dev-service-ports.sh | 132 +++++++++++++++++++++++++++--- 2 files changed, 160 insertions(+), 17 deletions(-) diff --git a/scripts/lib/dev-service-env.sh b/scripts/lib/dev-service-env.sh index 8c7e9738410..f8b9c2d6fff 100644 --- a/scripts/lib/dev-service-env.sh +++ b/scripts/lib/dev-service-env.sh @@ -10,12 +10,45 @@ validate_local_service_port() { # shellcheck disable=SC2016 if ! parsed=$(node -e ' + const { BlockList, isIP } = require("node:net"); + try { const url = new URL(process.argv[1]); - const host = url.hostname.toLowerCase(); - const isLocal = host === "localhost" || host === "127.0.0.1" || host === "[::1]"; - const port = url.port || process.argv[2]; - process.stdout.write(`${isLocal ? "local" : "remote"}\t${port}`); + const rawHost = url.hostname.toLowerCase(); + const dnsHost = rawHost.endsWith(".") ? rawHost.slice(0, -1) : rawHost; + + // Reparse through a special-scheme URL to canonicalize legacy numeric + // IPv4 forms such as 127.1, 0177.0.0.1, and 2130706433. The original + // postgres:/redis: URLs are non-special and preserve those spellings. + const canonicalHost = new URL(`http://${rawHost}/`) + .hostname + .replace(/^\[|\]$/g, ""); + const family = isIP(canonicalHost); + const addressType = family === 4 ? "ipv4" : "ipv6"; + + const loopback = new BlockList(); + loopback.addSubnet("127.0.0.0", 8, "ipv4"); + loopback.addAddress("::1", "ipv6"); + loopback.addSubnet("::ffff:127.0.0.0", 104, "ipv6"); + + const unspecified = new BlockList(); + unspecified.addAddress("0.0.0.0", "ipv4"); + unspecified.addAddress("::", "ipv6"); + unspecified.addAddress("::ffff:0.0.0.0", "ipv6"); + + const isLocalName = dnsHost === "localhost" || dnsHost.endsWith(".localhost"); + const isLoopbackAddress = family !== 0 && loopback.check(canonicalHost, addressType); + const isUnspecifiedAddress = family !== 0 && unspecified.check(canonicalHost, addressType); + const protocolDefault = url.protocol === "https:" ? "443" + : url.protocol === "http:" ? "80" + : process.argv[2]; + const port = url.port || protocolDefault; + const location = isUnspecifiedAddress + ? "unspecified" + : isLocalName || isLoopbackAddress + ? "local" + : "remote"; + process.stdout.write(`${location}\t${port}`); } catch { process.exit(1); } @@ -26,6 +59,10 @@ validate_local_service_port() { local location="${parsed%%$'\t'*}" local actual_port="${parsed#*$'\t'}" + if [[ "${location}" == "unspecified" ]]; then + echo "${service} URL uses an unspecified local address; use localhost, a loopback address, or an explicit remote host" >&2 + return 1 + fi if [[ "${location}" == "local" && "${actual_port}" != "${expected_port}" ]]; then echo "${service} URL uses local port ${actual_port}, but its Compose host port is ${expected_port}" >&2 return 1 diff --git a/scripts/test-dev-service-ports.sh b/scripts/test-dev-service-ports.sh index e6a34cbb88e..eb3447706b1 100755 --- a/scripts/test-dev-service-ports.sh +++ b/scripts/test-dev-service-ports.sh @@ -25,19 +25,76 @@ validate_local_service_port "Redis" "redis://127.0.0.1:16379" "16379" "6379" validate_local_service_port "MinIO" "http://[::1]:19000" "19000" "80" validate_local_service_port "External Postgres" "postgres://db.example.com:5432/buzz" "15432" "5432" -validation_error="${TMP_DIR}/validation-error" -if validate_local_service_port \ - "Postgres" \ - "postgres://buzz:do-not-print@localhost:5432/buzz" \ +local_aliases=( + "localhost" + "localhost." + "agent.localhost" + "agent.localhost." + "127.0.0.1" + "127.1" + "127.255.255.254" + "2130706433" + "0177.0.0.1" + "0x7f000001" + "[::1]" + "[::ffff:127.0.0.1]" + "[::ffff:7f00:1]" +) +for host in "${local_aliases[@]}"; do + validate_local_service_port \ + "Postgres" \ + "postgres://buzz:secret@${host}:15432/buzz" \ + "15432" \ + "5432" +done + +assert_validation_rejected() { + local description="$1" + local url="$2" + local expected_port="$3" + local default_url_port="$4" + local validation_error="${TMP_DIR}/validation-error" + + if validate_local_service_port \ + "${description}" \ + "${url}" \ + "${expected_port}" \ + "${default_url_port}" 2> "${validation_error}"; then + echo "${description} unexpectedly passed validation" >&2 + exit 1 + fi + if grep -q 'do-not-print' "${validation_error}"; then + echo "${description} leaked URL credentials" >&2 + exit 1 + fi +} + +for host in "${local_aliases[@]}"; do + assert_validation_rejected \ + "mismatched local alias ${host}" \ + "postgres://buzz:do-not-print@${host}:5432/buzz" \ + "15432" \ + "5432" +done + +assert_validation_rejected \ + "unspecified IPv4 address" \ + "postgres://buzz:do-not-print@0.0.0.0:15432/buzz" \ "15432" \ - "5432" 2> "${validation_error}"; then - echo "mismatched local URL and host port unexpectedly passed validation" >&2 - exit 1 -fi -if grep -q 'do-not-print' "${validation_error}"; then - echo "port validation leaked URL credentials" >&2 - exit 1 -fi + "5432" +assert_validation_rejected \ + "unspecified IPv6 address" \ + "postgres://buzz:do-not-print@[::]:15432/buzz" \ + "15432" \ + "5432" +assert_validation_rejected \ + "IPv4-mapped unspecified address" \ + "postgres://buzz:do-not-print@[::ffff:0.0.0.0]:15432/buzz" \ + "15432" \ + "5432" + +# Special-scheme URLs use their protocol default when the port is omitted. +validate_local_service_port "HTTPS MinIO" "https://localhost" "443" "80" # The shared relay launcher may honor Compose host-port overrides, but it must # never apply schemas to or start against service URLs and credentials inherited @@ -220,7 +277,7 @@ cat > "${setup_probe_root}/mock-hermit" <<'EOF' set -euo pipefail # Mimic the generated Hermit node shim closely enough for the URL parser used -# by dev-service-env.sh. The probe only uses the three default local URLs. +# by dev-service-env.sh. The direct helper matrix above exercises real Node. url="${7}" default_port="${8}" case "${url}" in @@ -273,4 +330,53 @@ if ! grep -Fqx 'dev-setup reached Compose' "${setup_probe_output}"; then exit 1 fi +# A mismatched alternate loopback spelling must fail before setup starts +# Compose or migrations. The direct helper matrix above protects the real +# classifier; this probe protects its ordering in the setup boundary. +cat > "${setup_probe_root}/.env" <<'EOF' +PGPORT=15432 +DATABASE_URL=postgres://buzz:do-not-print@localhost.:5432/buzz +EOF + +setup_side_effects="${TMP_DIR}/setup-side-effects" +cat > "${setup_probe_root}/bin/just" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +echo compose >> "${SETUP_SIDE_EFFECTS}" +exit 1 +EOF +chmod +x "${setup_probe_root}/bin/just" +cat > "${setup_probe_root}/bin/cargo" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +echo migration >> "${SETUP_SIDE_EFFECTS}" +exit 1 +EOF +chmod +x "${setup_probe_root}/bin/cargo" + +if ( + cd "${setup_probe_root}" + PATH="${setup_probe_root}/mock-bin:/usr/bin:/bin" \ + HERMIT_EXE="${setup_probe_root}/mock-hermit" \ + SETUP_SIDE_EFFECTS="${setup_side_effects}" \ + ./scripts/dev-setup.sh +) > "${setup_probe_output}" 2>&1; then + echo "dev-setup unexpectedly accepted a mismatched local alias" >&2 + exit 1 +fi +if ! grep -Fq 'DATABASE_URL URL uses local port 5432, but its Compose host port is 15432' "${setup_probe_output}"; then + cat "${setup_probe_output}" >&2 + echo "dev-setup did not report the alternate-loopback port mismatch" >&2 + exit 1 +fi +if [[ -e "${setup_side_effects}" ]]; then + cat "${setup_side_effects}" >&2 + echo "dev-setup reached Compose or migrations after rejecting its service URL" >&2 + exit 1 +fi +if grep -q 'do-not-print' "${setup_probe_output}"; then + echo "dev-setup leaked URL credentials in its validation error" >&2 + exit 1 +fi + echo "Dev service port checks passed"