fix(backup): prevent stale and incomplete database restores

lib/common.sh

@@ -35,6 +35,62 @@ die() {
  exit 1
 }
 
+# Normalize redundant leading and trailing slashes without requiring the path
+# to exist. This keeps legacy SQLite URLs containing five slashes compatible
+# while producing the same path used by normal four-slash absolute URLs.
+normalize_posix_path() {
+ local path="$1"
+
+ while [[ "$path" == //* ]]; do
+ path="${path#/}"
+ done
+ while [[ "$path" != "/" && "$path" == */ ]]; do
+ path="${path%/}"
+ done
+
+ printf '%s\n' "$path"
+}
+
+# Return the filesystem path represented by a SQLAlchemy SQLite URL.
+# sqlite:///relative.db -> relative.db
+# sqlite:////absolute/db -> /absolute/db
+# sqlite://///absolute/db -> /absolute/db (legacy installer output)
+sqlite_database_path_from_url() {
+ local url="$1"
+ local url_part=""
+ local path=""
+
+ [[ "$url" =~ ^sqlite[^:]*:// ]] || return 1
+
+ url_part="${url#*://}"
+ url_part="${url_part%%\?*}"
+ url_part="${url_part%%#*}"
+
+ if [[ "$url_part" == //* ]]; then
+ path="/${url_part#//}"
+ elif [[ "$url_part" == /* ]]; then
+ path="${url_part#/}"
+ else
+ path="$url_part"
+ fi
+
+ normalize_posix_path "$path"
+}
+
+# Build a SQLAlchemy URL for an absolute SQLite database path. Stripping the
+# path's leading slash before adding the URL prefix guarantees exactly four
+# slashes after the scheme separator.
+sqlite_absolute_database_url() {
+ local driver="$1"
+ local path=""
+
+ path=$(normalize_posix_path "$2")
+ [[ "$driver" =~ ^sqlite([+][A-Za-z0-9_]+)?$ ]] || return 1
+ [[ "$path" == /* ]] || return 1
+
+ printf '%s:////%s\n' "$driver" "${path#/}"
+}
+
 # Ensure a secret-bearing file (e.g. .env, TLS private key) is only readable by
 # its owner. Creates the file with 0600 if it is missing so callers can harden
 # it *before* writing secrets; tightens it to 0600 if it already exists. A

lib/pasarguard-restore.sh

@@ -35,7 +35,27 @@ postgres_dump_looks_restorable() {
  local dump_file="$1"
  [ -s "$dump_file" ] || return 1
  # Require at least one real schema/data statement, not just comments/SET.
- grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file"
+ grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file" || return 1
+ # pg_dump writes this only after completing the output. Requiring it keeps a
+ # dump truncated by a full disk or interrupted process from being accepted.
+ grep -qE '^-- PostgreSQL database dump complete([[:space:]]*)$' "$dump_file"
+}
+
+# pg_dumpall uses a different completion marker for the globals-only file.
+postgres_globals_dump_looks_complete() {
+ local dump_file="$1"
+ [ -s "$dump_file" ] || return 1
+ grep -qE '^-- PostgreSQL database cluster dump complete([[:space:]]*)$' "$dump_file"
+}
+
+# Both mysqldump and mariadb-dump emit a completion marker after a successful
+# plain-SQL dump. This accepts either tool while rejecting empty and truncated
+# files before they can be archived or restored.
+mysql_dump_looks_restorable() {
+ local dump_file="$1"
+ [ -s "$dump_file" ] || return 1
+ grep -qE '^-- (MySQL|MariaDB) dump ' "$dump_file" || return 1
+ grep -qE '^-- Dump completed on ' "$dump_file"
 }
 
 # Detect the dump layout inside an extracted backup directory.
@@ -253,6 +273,10 @@ restore_command() {
  local current_sqlalchemy_url=""
  local current_mysql_root_password=""
  local sqlite_basename=""
+ local sqlite_backup_source=""
+ local sqlite_safety_backup=""
+ local restore_timestamp=""
+ restore_timestamp=$(date +%Y%m%d%H%M%S)
 
  redact_database_url() {
  local url="$1"
@@ -684,16 +708,11 @@ restore_command() {
  if [[ "$SQLALCHEMY_DATABASE_URL" =~ ^sqlite ]]; then
  db_type="sqlite"
  colorized_echo green "✓ Detected SQLite database"
- local sqlite_url_part="${SQLALCHEMY_DATABASE_URL#*://}"
- sqlite_url_part="${sqlite_url_part%%\?*}"
- sqlite_url_part="${sqlite_url_part%%#*}"
-
- if [[ "$sqlite_url_part" =~ ^//(.*)$ ]]; then
- sqlite_file="/${BASH_REMATCH[1]}"
- elif [[ "$sqlite_url_part" =~ ^/(.*)$ ]]; then
- sqlite_file="/${BASH_REMATCH[1]}"
- else
- sqlite_file="$sqlite_url_part"
+ if ! sqlite_file=$(sqlite_database_path_from_url "$SQLALCHEMY_DATABASE_URL") || [ -z "$sqlite_file" ]; then
+ colorized_echo red "Invalid SQLite SQLALCHEMY_DATABASE_URL in backup; expected sqlite[+driver]:// followed by a database path."
+ echo "Invalid SQLite SQLALCHEMY_DATABASE_URL: $(redact_database_url "$SQLALCHEMY_DATABASE_URL")" >>"$log_file"
+ rm -rf "$temp_restore_dir"
+ exit 1
  fi
  colorized_echo blue "Database file: $sqlite_file"
  elif [[ "$SQLALCHEMY_DATABASE_URL" =~ ^(mysql|mariadb|postgresql)[^:]*:// ]]; then
@@ -816,39 +835,53 @@ restore_command() {
  case $db_type in
  sqlite)
  sqlite_basename=$(basename "$sqlite_file")
- local backup_source=""
 
  if [ -f "$temp_restore_dir/$sqlite_basename" ]; then
-backup_source="$temp_restore_dir/$sqlite_basename"
+sqlite_backup_source="$temp_restore_dir/$sqlite_basename"
  elif [ -f "$temp_restore_dir/db_backup.sqlite" ]; then
-backup_source="$temp_restore_dir/db_backup.sqlite"
+sqlite_backup_source="$temp_restore_dir/db_backup.sqlite"
  fi
 
- if [ -z "$backup_source" ]; then
+ if [ -z "$sqlite_backup_source" ]; then
  colorized_echo red "SQLite backup file not found in backup archive (looked for $sqlite_basename or db_backup.sqlite)."
  rm -rf "$temp_restore_dir"
  exit 1
  fi
 
- rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true
-
- if [ -f "$sqlite_file" ]; then
- cp "$sqlite_file" "${sqlite_file}.backup.$(date +%Y%m%d%H%M%S)" 2>>"$log_file"
+ if ! command -v sqlite3 >/dev/null 2>&1; then
+ detect_os
+ try_install_package sqlite3 || true
  fi
-
- if cp "$backup_source" "$sqlite_file" 2>>"$log_file"; then
- colorized_echo green "SQLite database restored successfully."
- else
- colorized_echo red "Failed to restore SQLite database."
- echo "SQLite restore failed" >>"$log_file"
+ if ! command -v sqlite3 >/dev/null 2>&1; then
+ colorized_echo red "sqlite3 is required to validate the SQLite snapshot before restore. Install sqlite3 and run the restore again."
+ echo "sqlite3 unavailable; cannot validate $sqlite_backup_source" >>"$log_file"
  rm -rf "$temp_restore_dir"
  exit 1
  fi
+ if ! sqlite_snapshot_looks_restorable "$sqlite_backup_source"; then
+ colorized_echo red "SQLite backup is corrupt or incomplete; aborting before replacing the current database."
+ echo "SQLite snapshot validation failed for $sqlite_backup_source" >>"$log_file"
+ rm -rf "$temp_restore_dir"
+ exit 1
+ fi
+
+ if [ -f "$sqlite_file" ]; then
+ sqlite_safety_backup="$backup_dir/sqlite_before_restore_${restore_timestamp}_${sqlite_basename}"
+ if ! sqlite3 "$sqlite_file" ".backup '$sqlite_safety_backup'" >>"$log_file" 2>&1; then
+ colorized_echo red "Failed to create a safety snapshot of the current SQLite database; restore aborted."
+ echo "SQLite safety snapshot failed: $sqlite_file -> $sqlite_safety_backup" >>"$log_file"
+ rm -f "$sqlite_safety_backup"
+ rm -rf "$temp_restore_dir"
+ exit 1
+ fi
+ colorized_echo blue "Current SQLite database saved to $sqlite_safety_backup"
+ fi
  ;;
 
  mariadb|mysql)
- if [ ! -f "$temp_restore_dir/db_backup.sql" ]; then
- colorized_echo red "Database backup file not found in backup archive."
+ if ! mysql_dump_looks_restorable "$temp_restore_dir/db_backup.sql"; then
+ colorized_echo red "Database backup is missing, truncated, or invalid; aborting before restore."
+ echo "MySQL/MariaDB dump validation failed for $temp_restore_dir/db_backup.sql" >>"$log_file"
  rm -rf "$temp_restore_dir"
  exit 1
  fi
@@ -968,6 +1001,13 @@ restore_command() {
  exit 1
  fi
 
+ if [ "$pg_layout" = "multi" ] && ! postgres_backup_looks_restorable "$temp_restore_dir" "$db_name"; then
+ colorized_echo red "Multi-database backup is incomplete or does not contain the configured database; aborting before restore."
+ echo "Multi-database dump validation failed for $temp_restore_dir/pg_dump" >>"$log_file"
+ rm -rf "$temp_restore_dir"
+ exit 1
+ fi
+
  if [ "$pg_layout" = "single" ]; then
  # Verify backup file is not empty and is readable
  if [ ! -s "$temp_restore_dir/db_backup.sql" ]; then
@@ -1155,13 +1195,29 @@ restore_command() {
  exit 1
  fi
  if [ "$db_type" = "sqlite" ] && [ -n "${sqlite_file:-}" ]; then
- rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true
+ rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true
  fi
  colorized_echo green "Data directory restored to $DATA_DIR."
  else
  colorized_echo yellow "No pasarguard_data directory found in backup. Skipping data restore."
  fi
 
+ # The data directory in legacy archives may contain a raw SQLite main file
+ # and WAL. Apply the consistent snapshot only after that directory has been
+ # restored so the raw copy can never overwrite the authoritative backup.
+ if [ "$db_type" = "sqlite" ]; then
+ mkdir -p "$(dirname "$sqlite_file")"
+ rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true
+ if cp "$sqlite_backup_source" "$sqlite_file" 2>>"$log_file"; then
+ colorized_echo green "SQLite database restored successfully."
+ else
+ colorized_echo red "Failed to restore SQLite database."
+ echo "SQLite restore failed" >>"$log_file"
+ rm -rf "$temp_restore_dir"
+ exit 1
+ fi
+ fi
+
  # Restore app directory files (full app backup support)
  colorized_echo blue "Restoring app directory files..."
  if [ -d "$temp_restore_dir" ]; then

pasarguard.sh

@@ -1067,7 +1067,8 @@ install_pasarguard() {
  db_driver_scheme="sqlite"
  fi
 
- sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${db_driver_scheme}:////${DATA_DIR}/db.sqlite3\"~" "$APP_DIR/.env"
+ SQLALCHEMY_DATABASE_URL=$(sqlite_absolute_database_url "$db_driver_scheme" "$DATA_DIR/db.sqlite3")
+ sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${SQLALCHEMY_DATABASE_URL}\"~" "$APP_DIR/.env"
 
  fi
 

tests/backup_restore_roundtrip.sh

@@ -57,9 +57,21 @@ EXTRACTED_BACKUP_DIR=""
 COMBINED_BACKUP_ARCHIVE=""
 MULTIPART_SPLIT_SIZE_BYTES=2048
 MULTIPART_SPLIT_THRESHOLD_BYTES=3072
+SQLITE_HOLDER_PID=""
+SQLITE_HOLDER_READY="$WORK_DIR/sqlite-holder.ready"
+SQLITE_HOLDER_STOP="$WORK_DIR/sqlite-holder.stop"
+
+stop_sqlite_holder() {
+ if [ -n "$SQLITE_HOLDER_PID" ] && kill -0 "$SQLITE_HOLDER_PID" 2>/dev/null; then
+ touch "$SQLITE_HOLDER_STOP"
+ wait "$SQLITE_HOLDER_PID"
+ fi
+ SQLITE_HOLDER_PID=""
+}
 
 cleanup() {
  local exit_code=$?
+ stop_sqlite_holder
  if [ "$exit_code" -ne 0 ] && [ -d "$WORK_DIR" ]; then
  while IFS= read -r log_path; do
  [ -n "$log_path" ] || continue
@@ -180,6 +192,7 @@ write_sqlite_env() {
  cat >"$ENV_FILE" <<EOF
 BACKUP_SERVICE_ENABLED=false
 RESTORE_TEST_FLAG=$EXPECTED_ENV_FLAG
+# Keep the legacy five-slash URL here to verify existing installations.
 SQLALCHEMY_DATABASE_URL="sqlite:////$DATA_DIR/db.sqlite3"
 EOF
 }
@@ -275,10 +288,36 @@ record_original_file_hashes() {
 }
 
 setup_sqlite_db() {
- sqlite3 "$DATA_DIR/db.sqlite3" <<EOF
-CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL);
-INSERT INTO ci_roundtrip (id, value) VALUES (1, '$EXPECTED_DB_VALUE');
-EOF
+ python3 - "$DATA_DIR/db.sqlite3" "$SQLITE_HOLDER_READY" "$SQLITE_HOLDER_STOP" "$EXPECTED_DB_VALUE" <<'PY' &
+import sqlite3
+import sys
+import time
+from pathlib import Path
+
+db_path, ready_path, stop_path, expected_value = sys.argv[1:]
+connection = sqlite3.connect(db_path)
+connection.execute("PRAGMA journal_mode=WAL")
+connection.execute("PRAGMA wal_autocheckpoint=0")
+connection.execute("CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL)")
+connection.execute("INSERT INTO ci_roundtrip (id, value) VALUES (1, 'old-checkpoint')")
+connection.commit()
+connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+connection.execute("UPDATE ci_roundtrip SET value = ? WHERE id = 1", (expected_value,))
+connection.commit()
+Path(ready_path).touch()
+while not Path(stop_path).exists():
+ time.sleep(0.05)
+connection.close()
+PY
+ SQLITE_HOLDER_PID=$!
+ wait_for_command 50 test -f "$SQLITE_HOLDER_READY"
+
+ # A valid SQLite file with the same basename in APP_DIR used to overwrite
+ # the authoritative snapshot while app files were copied into staging.
+ # Keep this collision valid (not garbage) so integrity checks alone cannot
+ # distinguish it from the live DATA_DIR database.
+ sqlite3 "$APP_DIR/db.sqlite3" \
+ "CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL); INSERT INTO ci_roundtrip VALUES (1, 'stale-app-dir-copy');"
 }
 
 sqlite_query() {
@@ -287,6 +326,7 @@ sqlite_query() {
 
 mutate_sqlite_db() {
  sqlite3 "$DATA_DIR/db.sqlite3" "UPDATE ci_roundtrip SET value = 'mutated' WHERE id = 1;"
+ printf 'stale rollback journal\n' >"$DATA_DIR/db.sqlite3-journal"
 }
 
 setup_mysql_container() {
@@ -447,11 +487,7 @@ verify_backup_archive_contents() {
  expected_files=$'.env\ndb_backup.sql\ndocker-compose.yml\npasarguard_data/\npasarguard_data/payload.bin\npasarguard_data/sentinel.txt'
  fi
 
- if [ "$DB_TYPE" = "sqlite" ]; then
- assert_zip_contains_required_files "$archive_to_verify" "$expected_files"
- else
- assert_zip_contains_exact_files "$archive_to_verify" "$expected_files"
- fi
+ assert_zip_contains_exact_files "$archive_to_verify" "$expected_files"
  assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/.env" | awk '{print $1}')" "$ORIGINAL_ENV_SHA" "Backed up .env contents changed."
  assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/docker-compose.yml" | awk '{print $1}')" "$ORIGINAL_COMPOSE_SHA" "Backed up docker-compose.yml contents changed."
  assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/pasarguard_data/sentinel.txt" | awk '{print $1}')" "$ORIGINAL_SENTINEL_SHA" "Backed up sentinel.txt contents changed."
@@ -460,9 +496,12 @@ verify_backup_archive_contents() {
  if [ "$DB_TYPE" = "sqlite" ]; then
  assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/$sqlite_basename"
  assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Backed up SQLite database logical contents changed."
- if [ -f "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ]; then
- assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename"
- assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Archived SQLite data-dir database logical contents changed."
+ if [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ] || \
+ [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-wal" ] || \
+ [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-shm" ] || \
+ [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-journal" ]; then
+ printf 'SQLite database or WAL/SHM/journal leaked into the raw data-directory copy.\n' >&2
+ exit 1
  fi
  elif [ "$DB_TYPE" = "postgresql" ] || [ "$DB_TYPE" = "timescaledb" ]; then
  assert_file_contains "$EXTRACTED_BACKUP_DIR/pg_dump/db-001.sql" "ci_roundtrip"
@@ -484,6 +523,20 @@ verify_restored_files() {
  if [ "$DB_TYPE" = "sqlite" ]; then
  assert_sqlite_integrity "$DATA_DIR/db.sqlite3"
  assert_equals "$(sqlite_dump_sha "$DATA_DIR/db.sqlite3")" "$ORIGINAL_SQLITE_DUMP_SHA" "SQLite database logical contents were not restored from backup."
+ if [ -e "$DATA_DIR/db.sqlite3-journal" ]; then
+ printf 'A stale SQLite rollback journal survived the restore.\n' >&2
+ exit 1
+ fi
+
+ local sqlite_safety_backup=""
+ sqlite_safety_backup=$(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'sqlite_before_restore_*_db.sqlite3' | sort | tail -n 1)
+ if [ -z "$sqlite_safety_backup" ]; then
+ printf 'The pre-restore SQLite safety snapshot did not survive the data-directory restore.\n' >&2
+ exit 1
+ fi
+ assert_sqlite_integrity "$sqlite_safety_backup"
+ assert_equals "$(sqlite3 "$sqlite_safety_backup" 'SELECT value FROM ci_roundtrip WHERE id = 1;')" \
+ "mutated" "Pre-restore SQLite safety snapshot did not preserve the replaced database."
  fi
 }
 
@@ -631,6 +684,7 @@ main() {
  prepare_case
  backup_command
  verify_backup_created
+ stop_sqlite_holder
  mutate_database_after_backup
  mutate_files_after_backup
  run_restore