From 3b0f6ee53f7c387eca1fa6711041664b78fc9997 Mon Sep 17 00:00:00 2001 From: hurricane1026 Date: Tue, 8 Sep 2026 21:48:36 +0800 Subject: [PATCH] test(pagestore): exercise portable bootstrap installation crashes --- .github/workflows/pagestore-test.yml | 3 + contrib/pagestore/MVP_COMPLETION_PLAN.md | 17 +- contrib/pagestore/MVP_STATUS.md | 10 +- .../harness/tests/bootstrap_install_oracle.py | 419 ++++++++++++++++++ .../tests/test_bootstrap_install_oracle.py | 205 +++++++++ contrib/pagestore/meson.build | 6 + contrib/pagestore/mvp_golden_test.sh | 84 +++- contrib/pagestore/pagestore.c | 101 ++++- contrib/pagestore/pagestore_fault_points.def | 4 + 9 files changed, 825 insertions(+), 24 deletions(-) create mode 100644 contrib/pagestore/harness/tests/bootstrap_install_oracle.py create mode 100644 contrib/pagestore/harness/tests/test_bootstrap_install_oracle.py diff --git a/.github/workflows/pagestore-test.yml b/.github/workflows/pagestore-test.yml index 34d974f7c0ae1..0ee7e6c278b0c 100644 --- a/.github/workflows/pagestore-test.yml +++ b/.github/workflows/pagestore-test.yml @@ -110,6 +110,9 @@ jobs: working-directory: contrib/pagestore run: ./pagestore_fault_test + - name: Run portable bootstrap install oracle unit tests + run: python3 -B contrib/pagestore/harness/tests/test_bootstrap_install_oracle.py + - name: Run named fault recovery harness working-directory: contrib/pagestore run: | diff --git a/contrib/pagestore/MVP_COMPLETION_PLAN.md b/contrib/pagestore/MVP_COMPLETION_PLAN.md index 24fda75069200..3669eae93bddb 100644 --- a/contrib/pagestore/MVP_COMPLETION_PLAN.md +++ b/contrib/pagestore/MVP_COMPLETION_PLAN.md @@ -785,7 +785,8 @@ restartpoint plans pause the checkpointer child after relation-page sync/before marker write and after marker sync, then stop and recover the whole materializer. The prepared-receipt/service-restore branch slice and the POSIX image-layer create/write/seal/manifest-ADD publication slice are also covered. -Branch bootstrap/install, manifest replacement, reclaim, and GC H1 cases remain. +Portable bootstrap/install is covered by the golden scenario's installer +crash/retry matrix. Manifest replacement, reclaim, and GC H1 cases remain. Deliverables: @@ -831,8 +832,8 @@ remain separate gates. ### H1. Compose process-level crash scenarios Status: **materializer replay/restartpoint, branch prepared-receipt/service- -restore, and POSIX image-layer publication slices implemented; branch -bootstrap/install, manifest replacement, reclaim, and GC cases remain and +restore, portable bootstrap/install, and POSIX image-layer publication slices +implemented; manifest replacement, reclaim, and GC cases remain and depend on H0/R2-R5**. Required scenario families: @@ -844,6 +845,16 @@ Required scenario families: orphan reconciliation, and timeline deletion; - daemon, writer, materializer, and branch-compute restart combinations. +The portable install slice in `mvp_golden_test.sh` targets an offline same-build, +default-tablespace skeleton. Four named installer-backend aborts cover maps +installed, the pg_xact remove/rename gap, and both sides of final manifest +publication. Each case checks the exact fault report and backend exit, +unchanged prepared inputs and restored control, startup rejection before +publication, full artifact recovery on retry, and byte-idempotent reinstall. +The resulting branch must pass golden SQL fork-point, parent/child isolation, +and restart checks. Power-loss recovery and concurrent installers/service +managers are outside this process-abort contract. + The materializer slice is split into two focused plans: one pauses after relation-page store sync and before marker write, and one pauses after marker store sync and before retention advance. The pause is reported by the named diff --git a/contrib/pagestore/MVP_STATUS.md b/contrib/pagestore/MVP_STATUS.md index 718435c033317..833ebb323e0b9 100644 --- a/contrib/pagestore/MVP_STATUS.md +++ b/contrib/pagestore/MVP_STATUS.md @@ -200,6 +200,14 @@ materializer, restores the normal writer, and advances the journal monotonically to `complete`. Bootstrap installation, layer recovery, and GC remain outside this slice. +Portable bootstrap installation has a separate golden-scenario crash slice: +installer-backend aborts after maps, in the pg_xact replacement gap, and on +both sides of final manifest publication. It checks startup rejection while +the manifest is absent, unchanged prepared inputs/control, exact artifact +recovery and idempotent retry, followed by branch SQL visibility and isolation. +The target stays offline under one installer; concurrent installation and +power-loss durability are not claimed by these process-abort tests. + The same prepare now captures every default-tablespace database relation map plus the global map under `RelationMappingLock` into one CRC-protected `pagestore_branch.bootstrap`. Its header binds the system identifier, logical @@ -347,7 +355,7 @@ remaining R6 queue-bound soak/tuning work. The POSIX image-layer publication slice is now covered by the declarative harness. Other crash boundaries remain outside this slice. Before declaring the MVP repeatable, add process-level fault scenarios around -branch bootstrap/install, manifest replacement, and retention/reclaim/GC, plus +manifest replacement and retention/reclaim/GC, plus a persisted-format fixture for restart/upgrade compatibility. ## Recommended sequence diff --git a/contrib/pagestore/harness/tests/bootstrap_install_oracle.py b/contrib/pagestore/harness/tests/bootstrap_install_oracle.py new file mode 100644 index 0000000000000..d7358ba07e06e --- /dev/null +++ b/contrib/pagestore/harness/tests/bootstrap_install_oracle.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Small, process-independent oracle for portable branch installation tests. + +The golden shell deliberately owns PostgreSQL and pg_ctl. This helper only +arms no processes and never starts a server; it records or checks the bytes +which must survive an installer backend crash. + +The public command line is intentionally boring so a shell test can use it +from any working directory:: + + bootstrap_install_oracle.py snapshot PREPARED TARGET OUT + bootstrap_install_oracle.py unchanged PREPARED TARGET SNAPSHOT + bootstrap_install_oracle.py installed PREPARED TARGET + bootstrap_install_oracle.py snapshot-installed TARGET OUT + bootstrap_install_oracle.py equal-installed TARGET SNAPSHOT + bootstrap_install_oracle.py report REPORT NAME OPERATION + +``snapshot`` covers the complete prepared directory and only +``TARGET/global/pg_control``. ``installed`` checks the portable bootstrap +artifact, relation maps encoded in that artifact, and the installed SLRUs. +The installed snapshot intentionally excludes logs, WAL, and pg_control. +Repeated ``snapshot-installed``/``equal-installed`` calls therefore provide +an idempotence check without making normal PostgreSQL startup noise part of +the oracle. +""" + +from __future__ import annotations + +import argparse +import glob +import hashlib +import json +import os +from pathlib import Path +import struct +import stat +import sys +from typing import Any, Iterable + + +FORMAT = 1 +MANIFEST = "pagestore_branch.manifest" +BOOTSTRAP = "pagestore_branch.bootstrap" +CONTROL = "global/pg_control" +REPORT_SCENARIO = "mvp-golden" +REPORT_SEED = 1 + +# The C structure is five uint64 fields followed by eight uint32 fields, +# including the CRC and manifest CRC. It is 72 bytes on native builds. +BOOTSTRAP_HEADER = struct.Struct("@QQQQQIIIIIIII") +BOOTSTRAP_MAP = struct.Struct("@II") +BOOTSTRAP_MAGIC = 0x50534242 +BOOTSTRAP_FORMAT = 1 + + +class OracleError(RuntimeError): + """An acceptance assertion failed.""" + + +def _fail(message: str) -> "NoReturn": + raise OracleError(message) + + +def _regular_bytes(path: Path) -> bytes: + try: + file_stat = path.lstat() + except FileNotFoundError: + _fail(f"missing regular file: {path}") + if not stat.S_ISREG(file_stat.st_mode): + _fail(f"expected non-symlink regular file: {path}") + try: + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError as exc: + _fail(f"could not open {path}: {exc}") + chunks: list[bytes] = [] + try: + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + finally: + os.close(fd) + return b"".join(chunks) + + +def _entry(path: Path) -> dict[str, Any]: + try: + file_stat = path.lstat() + except FileNotFoundError: + return {"kind": "missing"} + mode = file_stat.st_mode + if stat.S_ISREG(mode): + digest = hashlib.sha256(_regular_bytes(path)).hexdigest() + return {"kind": "file", "size": file_stat.st_size, "sha256": digest} + if stat.S_ISDIR(mode): + return {"kind": "dir"} + if stat.S_ISLNK(mode): + return {"kind": "symlink", "target": os.readlink(path)} + return {"kind": "other", "mode": mode & 0o7777} + + +def _relative(root: Path, path: Path) -> str: + value = os.path.relpath(path, root) + return "" if value == "." else value.replace(os.sep, "/") + + +def _snapshot_tree(root: Path) -> dict[str, dict[str, Any]]: + """Snapshot a tree without following symlinks.""" + + root = Path(root) + if not root.exists() and not root.is_symlink(): + _fail(f"missing snapshot root: {root}") + result: dict[str, dict[str, Any]] = {} + + def visit(path: Path) -> None: + relative = _relative(root, path) + item = _entry(path) + result[relative] = item + if item["kind"] != "dir": + return + try: + children = sorted(path.iterdir(), key=lambda child: child.name) + except OSError as exc: + _fail(f"could not enumerate {path}: {exc}") + for child in children: + visit(child) + + visit(root) + return result + + +def _snapshot_paths(root: Path, paths: Iterable[str]) -> dict[str, dict[str, Any]]: + """Snapshot exact relative paths, recursively for directory paths.""" + + root = Path(root) + result: dict[str, dict[str, Any]] = {} + for relative in sorted(set(paths)): + path = root / relative + item = _entry(path) + result[relative] = item + if item["kind"] == "dir": + for child, child_item in _snapshot_tree(path).items(): + if child == "": + continue + result[f"{relative}/{child}"] = child_item + return result + + +def _write_json(path: Path, value: Any) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp.{os.getpid()}") + data = json.dumps(value, sort_keys=True, indent=2).encode("utf-8") + b"\n" + with open(temporary, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + + +def _read_json(path: Path) -> Any: + try: + with open(path, "rb") as stream: + return json.load(stream) + except (OSError, ValueError) as exc: + _fail(f"could not read JSON snapshot {path}: {exc}") + + +def _snapshot_inputs(prepared: Path, target: Path) -> dict[str, Any]: + control = _snapshot_paths(target, [CONTROL]) + if control[CONTROL].get("kind") != "file": + _fail(f"target control file is not a regular file: {target / CONTROL}") + return { + "format": FORMAT, + "prepared": _snapshot_tree(prepared), + "target_pg_control": control, + } + + +def _assert_equal(label: str, expected: Any, actual: Any) -> None: + if expected != actual: + _fail(f"{label} changed") + + +def snapshot_inputs(prepared: Path, target: Path, output: Path) -> None: + _write_json(output, _snapshot_inputs(prepared, target)) + + +def unchanged_inputs(prepared: Path, target: Path, snapshot: Path) -> None: + saved = _read_json(snapshot) + if saved.get("format") != FORMAT: + _fail(f"unsupported snapshot format in {snapshot}") + _assert_equal("prepared artifacts or target pg_control", saved, + _snapshot_inputs(prepared, target)) + + +def _manifest(prepared: Path) -> tuple[bytes, dict[str, Any]]: + data = _regular_bytes(prepared / MANIFEST) + try: + value = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + _fail(f"invalid prepared branch manifest: {exc}") + if not isinstance(value, dict): + _fail("prepared branch manifest is not an object") + return data, value + + +def _commit_ts_required(manifest: dict[str, Any]) -> bool: + try: + oldest = int(manifest["oldest_commit_ts_xid"]) + newest = int(manifest["next_commit_ts_xid"]) + except (KeyError, TypeError, ValueError): + _fail("manifest has invalid commit-ts horizons") + if not (0 <= oldest <= 0xFFFFFFFF and 0 <= newest <= 0xFFFFFFFF): + _fail("manifest has out-of-range commit-ts horizons") + normal_oldest = 3 <= oldest <= 0xFFFFFFFF + normal_newest = 3 <= newest <= 0xFFFFFFFF + if not normal_oldest and not normal_newest: + return False + if not normal_oldest or not normal_newest: + _fail("manifest has a half-active commit-ts horizon") + # Match TransactionIdFollows(oldest, newest), including wraparound. + follows = oldest != newest and ((oldest - newest) & 0xFFFFFFFF) < 0x80000000 + if follows: + _fail("manifest has inverted commit-ts horizons") + return True + + +def _assert_tree_equal(prepared: Path, target: Path, relative: str) -> None: + expected = _snapshot_tree(prepared / relative) + actual = _snapshot_tree(target / relative) + _assert_equal(relative, expected, actual) + + +def _parse_bootstrap(data: bytes) -> list[tuple[int, bytes]]: + if len(data) < BOOTSTRAP_HEADER.size: + _fail("branch bootstrap artifact is truncated") + fields = BOOTSTRAP_HEADER.unpack_from(data) + artifact_size = fields[4] + magic = fields[5] + version = fields[6] + map_count = fields[9] + if magic != BOOTSTRAP_MAGIC or version != BOOTSTRAP_FORMAT: + _fail("branch bootstrap artifact has an invalid magic or format") + if artifact_size != len(data) or map_count < 2: + _fail("branch bootstrap artifact has an invalid size or map count") + cursor = BOOTSTRAP_HEADER.size + maps: list[tuple[int, bytes]] = [] + previous = -1 + for index in range(map_count): + if cursor + BOOTSTRAP_MAP.size > len(data): + _fail("branch bootstrap map table is truncated") + database_oid, size = BOOTSTRAP_MAP.unpack_from(data, cursor) + cursor += BOOTSTRAP_MAP.size + if database_oid <= previous or size == 0: + _fail("branch bootstrap maps are not strictly ordered") + if index == 0 and database_oid != 0: + _fail("branch bootstrap has no global relation map") + if cursor + size > len(data): + _fail("branch bootstrap relation map is truncated") + maps.append((database_oid, data[cursor:cursor + size])) + cursor += size + previous = database_oid + if cursor != len(data): + _fail("branch bootstrap has trailing bytes") + return maps + + +def installed(prepared: Path, target: Path) -> None: + manifest_data, manifest = _manifest(prepared) + bootstrap_data = _regular_bytes(prepared / BOOTSTRAP) + target_manifest = _regular_bytes(target / MANIFEST) + target_bootstrap = _regular_bytes(target / BOOTSTRAP) + _assert_equal("installed branch manifest", manifest_data, target_manifest) + _assert_equal("installed branch bootstrap", bootstrap_data, target_bootstrap) + + for database_oid, relation_map in _parse_bootstrap(bootstrap_data): + if database_oid == 0: + relative = "global/pg_filenode.map" + else: + relative = f"base/{database_oid}/pg_filenode.map" + _assert_equal(relative, relation_map, _regular_bytes(target / relative)) + + for relative in ("pg_xact", "pg_multixact/offsets", "pg_multixact/members"): + _assert_tree_equal(prepared, target, relative) + + commit_ts_target = target / "pg_commit_ts" + if _commit_ts_required(manifest): + _assert_tree_equal(prepared, target, "pg_commit_ts") + else: + if not commit_ts_target.is_dir() or commit_ts_target.is_symlink(): + _fail("inactive commit-ts target is not a real directory") + if any(commit_ts_target.iterdir()): + _fail("inactive commit-ts target is not empty") + + +def _installed_paths(target: Path) -> list[str]: + patterns = [ + MANIFEST, + BOOTSTRAP, + "global/pg_filenode.map", + "pg_xact", + "pg_commit_ts", + "pg_multixact", + ] + paths: list[str] = [] + for pattern in patterns: + matches = sorted(glob.glob(str(target / pattern))) + if not matches: + _fail(f"installed artifact is missing: {target / pattern}") + paths.extend(_relative(target, Path(match)) for match in matches) + maps = sorted(glob.glob(str(target / "base" / "*" / "pg_filenode.map"))) + if not maps: + _fail("installed database relation maps are missing") + paths.extend(_relative(target, Path(match)) for match in maps) + return sorted(set(paths)) + + +def snapshot_installed(target: Path, output: Path) -> None: + paths = _installed_paths(target) + _write_json(output, { + "format": FORMAT, + "paths": paths, + "snapshot": _snapshot_paths(target, paths), + }) + + +def equal_installed(target: Path, snapshot: Path) -> None: + saved = _read_json(snapshot) + if saved.get("format") != FORMAT or not isinstance(saved.get("paths"), list): + _fail(f"unsupported installed snapshot format in {snapshot}") + paths = [str(path) for path in saved["paths"]] + _assert_equal("installed artifact selection", sorted(paths), + _installed_paths(target)) + actual = _snapshot_paths(target, paths) + _assert_equal("installed artifacts", saved.get("snapshot"), actual) + + +def check_report(report_path: Path, name: str, operation: str) -> int: + data = _regular_bytes(report_path) + try: + records = [json.loads(line) for line in data.decode("utf-8").splitlines() + if line.strip()] + except (UnicodeDecodeError, ValueError) as exc: + _fail(f"invalid fault report: {exc}") + if len(records) != 1 or not isinstance(records[0], dict): + _fail("fault report does not contain exactly one JSON record") + record = records[0] + if (record.get("schema") != 1 or record.get("name") != name or + record.get("action") != "crash" or record.get("hit") != 1 or + record.get("scenario") != REPORT_SCENARIO or + record.get("seed") != REPORT_SEED or + record.get("operation") != operation): + _fail("fault report identity does not match the requested install point") + pid = record.get("pid") + if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: + _fail("fault report has no positive process PID") + return pid + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + command = commands.add_parser("snapshot") + command.add_argument("prepared", type=Path) + command.add_argument("target", type=Path) + command.add_argument("output", type=Path) + + command = commands.add_parser("unchanged") + command.add_argument("prepared", type=Path) + command.add_argument("target", type=Path) + command.add_argument("snapshot", type=Path) + + command = commands.add_parser("installed") + command.add_argument("prepared", type=Path) + command.add_argument("target", type=Path) + + command = commands.add_parser("snapshot-installed") + command.add_argument("target", type=Path) + command.add_argument("output", type=Path) + + command = commands.add_parser("equal-installed") + command.add_argument("target", type=Path) + command.add_argument("snapshot", type=Path) + + command = commands.add_parser("report") + command.add_argument("report", type=Path) + command.add_argument("name") + command.add_argument("operation") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.command == "snapshot": + snapshot_inputs(args.prepared, args.target, args.output) + elif args.command == "unchanged": + unchanged_inputs(args.prepared, args.target, args.snapshot) + elif args.command == "installed": + installed(args.prepared, args.target) + elif args.command == "snapshot-installed": + snapshot_installed(args.target, args.output) + elif args.command == "equal-installed": + equal_installed(args.target, args.snapshot) + elif args.command == "report": + print(check_report(args.report, args.name, args.operation)) + else: # pragma: no cover - argparse makes this unreachable + _fail(f"unknown command: {args.command}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except OracleError as exc: + print(f"bootstrap_install_oracle: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/contrib/pagestore/harness/tests/test_bootstrap_install_oracle.py b/contrib/pagestore/harness/tests/test_bootstrap_install_oracle.py new file mode 100644 index 0000000000000..69cbd4e837039 --- /dev/null +++ b/contrib/pagestore/harness/tests/test_bootstrap_install_oracle.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Unit tests for bootstrap_install_oracle.py; no PostgreSQL binaries needed.""" + +from __future__ import annotations + +import contextlib +import io +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +HELPER = Path(__file__).with_name("bootstrap_install_oracle.py") +SPEC = importlib.util.spec_from_file_location("bootstrap_install_oracle", HELPER) +assert SPEC is not None and SPEC.loader is not None +ORACLE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ORACLE) + + +class BootstrapInstallOracleTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.prepared = self.root / "prepared" + self.target = self.root / "target" + self.prepared.mkdir() + self.target.mkdir() + self._make_fixture() + + def tearDown(self) -> None: + self.temp.cleanup() + + def _write(self, root: Path, relative: str, data: bytes) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + def _make_fixture(self) -> None: + manifest = ( + b'{"format":2,"oldest_commit_ts_xid":"0",' + b'"next_commit_ts_xid":"0"}\n' + ) + maps = [(0, b"global-map\n"), (42, b"database-map\n")] + header_size = ORACLE.BOOTSTRAP_HEADER.size + artifact_size = header_size + sum( + ORACLE.BOOTSTRAP_MAP.size + len(data) for _oid, data in maps + ) + header = ORACLE.BOOTSTRAP_HEADER.pack( + 1, 2, 3, 4, artifact_size, ORACLE.BOOTSTRAP_MAGIC, + ORACLE.BOOTSTRAP_FORMAT, 1, 0, len(maps), 0, 0, 0, + ) + bootstrap = header + b"".join( + ORACLE.BOOTSTRAP_MAP.pack(oid, len(data)) + data + for oid, data in maps + ) + for root in (self.prepared, self.target): + self._write(root, ORACLE.MANIFEST, manifest) + self._write(root, ORACLE.BOOTSTRAP, bootstrap) + self._write(root, "pg_xact/0000", b"xact") + self._write(root, "pg_multixact/offsets/0000", b"offsets") + self._write(root, "pg_multixact/members/0000", b"members") + self._write(self.prepared, "extra/source-only", b"immutable") + self._write(self.target, ORACLE.CONTROL, b"control") + self._write(self.target, "global/pg_filenode.map", maps[0][1]) + self._write(self.target, "base/42/pg_filenode.map", maps[1][1]) + (self.target / "pg_commit_ts").mkdir() + + def test_snapshot_and_unchanged_cover_prepared_and_control(self) -> None: + snapshot = self.root / "inputs.json" + ORACLE.snapshot_inputs(self.prepared, self.target, snapshot) + ORACLE.unchanged_inputs(self.prepared, self.target, snapshot) + (self.prepared / "extra/source-only").write_bytes(b"changed") + with self.assertRaises(ORACLE.OracleError): + ORACLE.unchanged_inputs(self.prepared, self.target, snapshot) + + def test_target_control_mutation_is_rejected(self) -> None: + snapshot = self.root / "inputs.json" + ORACLE.snapshot_inputs(self.prepared, self.target, snapshot) + (self.target / ORACLE.CONTROL).write_bytes(b"control changed") + with self.assertRaises(ORACLE.OracleError): + ORACLE.unchanged_inputs(self.prepared, self.target, snapshot) + + def test_installed_checks_bootstrap_maps_slrus_and_inactive_commit_ts(self) -> None: + ORACLE.installed(self.prepared, self.target) + (self.target / "base/42/pg_filenode.map").write_bytes(b"wrong") + with self.assertRaises(ORACLE.OracleError): + ORACLE.installed(self.prepared, self.target) + + def test_missing_or_mutated_slru_is_rejected(self) -> None: + for relative in ( + "pg_xact/0000", + "pg_multixact/offsets/0000", + "pg_multixact/members/0000", + ): + with self.subTest(relative=relative): + path = self.target / relative + original = path.read_bytes() + path.unlink() + with self.assertRaises(ORACLE.OracleError): + ORACLE.installed(self.prepared, self.target) + path.write_bytes(original) + path.write_bytes(b"mutated") + with self.assertRaises(ORACLE.OracleError): + ORACLE.installed(self.prepared, self.target) + path.write_bytes(original) + + def test_commit_ts_activity_and_exact_contents_are_checked(self) -> None: + commit_ts = self.target / "pg_commit_ts" + self._write(self.target, "pg_commit_ts/unexpected", b"inactive") + with self.assertRaises(ORACLE.OracleError): + ORACLE.installed(self.prepared, self.target) + (commit_ts / "unexpected").unlink() + + active_manifest = ( + b'{"format":2,"oldest_commit_ts_xid":"10",' + b'"next_commit_ts_xid":"20"}\n' + ) + self._write(self.prepared, ORACLE.MANIFEST, active_manifest) + self._write(self.target, ORACLE.MANIFEST, active_manifest) + self._write(self.prepared, "pg_commit_ts/0000", b"commit-ts") + self._write(self.target, "pg_commit_ts/0000", b"commit-ts") + ORACLE.installed(self.prepared, self.target) + (commit_ts / "0000").write_bytes(b"wrong commit-ts") + with self.assertRaises(ORACLE.OracleError): + ORACLE.installed(self.prepared, self.target) + + def test_inverted_commit_ts_horizon_is_rejected(self) -> None: + with self.assertRaisesRegex(ORACLE.OracleError, "inverted commit-ts"): + ORACLE._commit_ts_required({ + "oldest_commit_ts_xid": "20", "next_commit_ts_xid": "10", + }) + self.assertTrue(ORACLE._commit_ts_required({ + "oldest_commit_ts_xid": "4294967290", "next_commit_ts_xid": "10", + })) + + def test_installed_snapshot_is_idempotent_and_excludes_control(self) -> None: + snapshot = self.root / "installed.json" + ORACLE.snapshot_installed(self.target, snapshot) + ORACLE.equal_installed(self.target, snapshot) + self._write(self.target, "server.log", b"normal startup noise") + self._write(self.target, ORACLE.CONTROL, b"control changed by startup") + ORACLE.equal_installed(self.target, snapshot) + (self.target / ORACLE.BOOTSTRAP).write_bytes(b"changed") + with self.assertRaises(ORACLE.OracleError): + ORACLE.equal_installed(self.target, snapshot) + + def test_added_database_map_is_detected_by_installed_snapshot(self) -> None: + snapshot = self.root / "installed.json" + ORACLE.snapshot_installed(self.target, snapshot) + self._write(self.target, "base/43/pg_filenode.map", b"unexpected map") + with self.assertRaises(ORACLE.OracleError): + ORACLE.equal_installed(self.target, snapshot) + + def test_report_returns_exact_positive_pid(self) -> None: + report = self.root / "report.jsonl" + record = { + "schema": 1, + "name": "branch_install.after_maps", + "action": "crash", + "scenario": "mvp-golden", + "seed": 1, + "operation": "install-after-maps", + "hit": 1, + "pid": 12345, + } + report.write_text(json.dumps(record) + "\n") + output = io.StringIO() + with contextlib.redirect_stdout(output): + pid = ORACLE.check_report( + report, "branch_install.after_maps", "install-after-maps" + ) + self.assertEqual(pid, 12345) + self.assertEqual(output.getvalue(), "") + + for field, value in ( + ("name", "branch_install.before_manifest"), + ("operation", "other-operation"), + ("hit", 2), + ("pid", 0), + ): + with self.subTest(field=field): + invalid = dict(record) + invalid[field] = value + report.write_text(json.dumps(invalid) + "\n") + with self.assertRaises(ORACLE.OracleError): + ORACLE.check_report( + report, "branch_install.after_maps", "install-after-maps" + ) + + report.write_text(json.dumps(record) + "\n" + json.dumps(record) + "\n") + with self.assertRaises(ORACLE.OracleError): + ORACLE.check_report( + report, "branch_install.after_maps", "install-after-maps" + ) + + with self.assertRaises(ORACLE.OracleError): + ORACLE._parse_bootstrap( + b"\0" * (ORACLE.BOOTSTRAP_HEADER.size - 1) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/pagestore/meson.build b/contrib/pagestore/meson.build index 1ffbf118ed765..76e9e8abfc4bb 100644 --- a/contrib/pagestore/meson.build +++ b/contrib/pagestore/meson.build @@ -529,6 +529,12 @@ if host_system != 'windows' suite: ['pagestore'], timeout: 30, ) + test('pagestore_bootstrap_install_oracle', + pagestore_harness_python, + args: [files('harness/tests/test_bootstrap_install_oracle.py')], + suite: ['pagestore'], + timeout: 30, + ) endif test('pagestore_harness_reader_visibility_plan', pagestore_harness_python, diff --git a/contrib/pagestore/mvp_golden_test.sh b/contrib/pagestore/mvp_golden_test.sh index a7f3071f5e1e3..033058d011bee 100755 --- a/contrib/pagestore/mvp_golden_test.sh +++ b/contrib/pagestore/mvp_golden_test.sh @@ -534,9 +534,6 @@ echo "ok - parent advanced and was durably materialized beyond the child fork" "$BRANCH" >/dev/null || fail "could not restore branch checkpoint control" branch_wal_segment_size=$(wal_segment_size "$BRANCH") || fail "could not read branch WAL segment size" -"${WP[@]}" -c "SELECT pagestore_ext.pagestore_install_prepared_branch_bootstrap( - '$PREPARED', '$BRANCH', 1, 0, '$checkpoint_redo', '$checkpoint_lsn', - '$fork_lsn');" >/dev/null || fail "could not install the portable branch bootstrap" # Remove the unrelated WAL segment created by the fresh initdb. The restored # cluster identity must fetch its checkpoint and all subsequent WAL from store. find "$BRANCH/pg_wal" -maxdepth 1 -type f -name '0000000*' -delete || @@ -561,6 +558,87 @@ recovery_target_action = 'promote' EOF touch "$BRANCH/recovery.signal" +# Exercise the portable installer in a real writer backend. Its process exits +# at the named boundary; the target remains offline until an unarmed retry. +# Each iteration also covers reinstalling an already completed target. +INSTALL_ORACLE="$(dirname "$0")/harness/tests/bootstrap_install_oracle.py" +install_branch_bootstrap() +{ + "${WP[@]}" -c "SELECT pagestore_ext.pagestore_install_prepared_branch_bootstrap( + '$PREPARED', '$BRANCH', 1, 0, '$checkpoint_redo', '$checkpoint_lsn', + '$fork_lsn');" +} +for install_phase in after_maps after_slru_remove before_manifest after_manifest; do + install_control="$TMPROOT/install-$install_phase" + mkdir -m 700 "$install_control" || fail "could not create install fault control" + printf 'arm\n' > "$install_control/arm" || fail "could not arm install fault" + python3 "$INSTALL_ORACLE" snapshot "$PREPARED" "$BRANCH" \ + "$install_control/before.json" || fail "could not snapshot install inputs" + "$BIN/pg_ctl" -D "$WRITER" -m fast -w stop >/dev/null 2>&1 || + fail "could not stop writer before install crash" + env -u PAGESTORE_TEST_FAULT_OPERATION_ID -u PAGESTORE_TEST_FAULT_WATCHDOG_MS \ + PAGESTORE_TEST_FAULT_NAME="branch_install.$install_phase" \ + PAGESTORE_TEST_FAULT_ACTION=crash PAGESTORE_TEST_FAULT_HIT=1 \ + PAGESTORE_TEST_FAULT_DIR="$install_control" \ + PAGESTORE_TEST_FAULT_SCENARIO=mvp-golden PAGESTORE_TEST_FAULT_SEED=1 \ + PAGESTORE_TEST_FAULT_OPERATION="install-$install_phase" \ + "$BIN/pg_ctl" -D "$WRITER" -l "$WRITER/writer.log" -w start \ + >/dev/null 2>&1 || fail "could not start fault-enabled installer host" + if install_branch_bootstrap > "$install_control/sql.log" 2>&1; then + fail "install fault was not reached: $install_phase" + fi + install_pid=$(python3 "$INSTALL_ORACLE" report "$install_control/report.jsonl" \ + "branch_install.$install_phase" "install-$install_phase") || + fail "install crash report does not match the requested boundary" + # psql's connection failure is insufficient proof: require the backend's + # exact PID and the canonical process-abort exit code in the server log. + for ((attempt=0; attempt<100; attempt++)); do + if grep -E "(client backend|server process) \(PID $install_pid\) exited with exit code 88" \ + "$WRITER/writer.log" >/dev/null; then + break + fi + sleep 0.1 + done + [ "$attempt" -lt 100 ] || fail "installer did not exit with fault status 88" + "$BIN/pg_ctl" -D "$WRITER" -m immediate -w stop >/dev/null 2>&1 || + fail "could not stop fault-enabled writer" + env -u PAGESTORE_TEST_FAULT_NAME -u PAGESTORE_TEST_FAULT_ACTION \ + -u PAGESTORE_TEST_FAULT_HIT -u PAGESTORE_TEST_FAULT_DIR \ + -u PAGESTORE_TEST_FAULT_SCENARIO -u PAGESTORE_TEST_FAULT_SEED \ + -u PAGESTORE_TEST_FAULT_OPERATION -u PAGESTORE_TEST_FAULT_OPERATION_ID \ + -u PAGESTORE_TEST_FAULT_WATCHDOG_MS \ + "$BIN/pg_ctl" -D "$WRITER" -l "$WRITER/writer.log" -w start \ + >/dev/null 2>&1 || fail "could not restart unarmed writer" + python3 "$INSTALL_ORACLE" unchanged "$PREPARED" "$BRANCH" \ + "$install_control/before.json" || fail "install crash changed its source or control" + if [ "$install_phase" != after_manifest ]; then + [ ! -e "$BRANCH/pagestore_branch.manifest" ] || + fail "incomplete install published a branch manifest" + if "$BIN/pg_ctl" -D "$BRANCH" -l "$install_control/startup.log" -w start \ + >/dev/null 2>&1; then + fail "partially installed branch was admitted" + fi + grep -F 'pagestore.timeline requires pagestore_branch.manifest' \ + "$install_control/startup.log" >/dev/null || + fail "partial branch failed for a reason other than the manifest fence" + [ ! -e "$BRANCH/postmaster.pid" ] || fail "failed branch startup remains live" + else + python3 "$INSTALL_ORACLE" installed "$PREPARED" "$BRANCH" || + fail "post-publication crash left an incomplete installation" + fi + install_branch_bootstrap >/dev/null || fail "portable install retry failed" + python3 "$INSTALL_ORACLE" installed "$PREPARED" "$BRANCH" || + fail "retried installation differs from prepared artifacts" + python3 "$INSTALL_ORACLE" snapshot-installed "$BRANCH" \ + "$install_control/installed.json" || fail "could not snapshot completed install" + install_branch_bootstrap >/dev/null || fail "repeated portable install failed" + python3 "$INSTALL_ORACLE" equal-installed "$BRANCH" \ + "$install_control/installed.json" || fail "repeated portable install is not idempotent" + python3 "$INSTALL_ORACLE" unchanged "$PREPARED" "$BRANCH" \ + "$install_control/before.json" || fail "install retry changed its source or control" + echo "ok - portable bootstrap crash/retry: $install_phase" +done + "$BIN/pg_ctl" -D "$BRANCH" -l "$BRANCH/branch.log" -w start >/dev/null 2>&1 || fail "independent branch compute did not boot" wait_branch_promotion || fail "portable branch recovery did not promote" diff --git a/contrib/pagestore/pagestore.c b/contrib/pagestore/pagestore.c index a15b63901359c..6c3788c4976d1 100644 --- a/contrib/pagestore/pagestore.c +++ b/contrib/pagestore/pagestore.c @@ -140,6 +140,11 @@ static recovery_start_hook_type prev_recovery_start_hook = NULL; static recovery_restartpoint_pre_control_hook_type prev_restartpoint_pre_control_hook = NULL; static recovery_restartpoint_flush_hook_type prev_restartpoint_flush_hook = NULL; +/* Fault probes below are scoped to the portable bootstrap installer. Legacy + * branch installs and reader installs must keep their existing fault surface. */ +static bool pagestore_portable_install_faults = false; +static bool pagestore_portable_install_faults_owned = false; + #define PS_MATERIALIZER_MARKER_MAGIC 0x50534d57 #define PS_MATERIALIZER_MARKER_VERSION 2 #define PS_MATERIALIZER_MARKER_BLOCK 3 @@ -10457,6 +10462,46 @@ pagestore_require_prepared_artifact(const char *prepared_dir, pagestore_require_regular_tree(path); } +/* + * Portable bootstrap faults are opt-in and process-abort-only. Do not call + * ps_fault_init() for ordinary installs: a materializer may already own the + * process-local fault state, and legacy/readers installs must not acquire a + * portable branch fault surface by accident. + */ +static void +pagestore_prepare_portable_install_faults(void) +{ + const char *name = getenv("PAGESTORE_TEST_FAULT_NAME"); + bool already_initialized; + + if (name == NULL || strncmp(name, "branch_install.", + strlen("branch_install.")) != 0) + return; + already_initialized = ps_fault_is_initialized(); + if (!already_initialized) + { + if (ps_fault_init(DataDir) != 0) + { + /* ps_fault_init marks the state initialized before validating the + * control protocol; do not strand that partial state across a + * retry after the ERROR below. */ + ps_fault_reset(); + ereport(ERROR, + (errmsg("could not initialize pagestore portable install fault controls"))); + } + pagestore_portable_install_faults_owned = true; + } +} + +static void +pagestore_portable_install_fault(PsFaultPoint point) +{ + if (pagestore_portable_install_faults && + ps_fault_probe(point) == PS_FAULT_PROBE_ERROR) + ereport(ERROR, + (errmsg("pagestore portable install fault probe failed"))); +} + /* * Absolutize (against the backend cwd, i.e. the data directory) and * canonicalize an install path so relative and absolute spellings of the same @@ -11132,6 +11177,8 @@ pagestore_install_prepared_dir(const char *prepared_dir, const char *target_dir, ereport(ERROR, (errcode_for_file_access(), errmsg("could not remove existing branch artifact \"%s\"", dst))); + if (strcmp(relpath, "pg_xact") == 0) + pagestore_portable_install_fault(PS_FAULT_POINT_BRANCH_INSTALL_AFTER_SLRU_REMOVE); if (rename(stage, dst) != 0) ereport(ERROR, (errcode_for_file_access(), @@ -11210,11 +11257,15 @@ pagestore_install_prepared_file(const char *prepared_dir, const char *target_dir (errcode_for_file_access(), errmsg("could not clear branch install staging file \"%s\": %m", stage))); copy_file(src, stage); + if (strcmp(relpath, "pagestore_branch.manifest") == 0) + pagestore_portable_install_fault(PS_FAULT_POINT_BRANCH_INSTALL_BEFORE_MANIFEST); if (durable_rename(stage, dst, ERROR) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not install branch artifact \"%s\": %m", dst))); fsync_fname(target_dir, true); + if (strcmp(relpath, "pagestore_branch.manifest") == 0) + pagestore_portable_install_fault(PS_FAULT_POINT_BRANCH_INSTALL_AFTER_MANIFEST); } static void @@ -11381,7 +11432,8 @@ pagestore_install_branch_bootstrap_maps(const char *prepared_dir, (int) entry.size); } pagestore_install_prepared_file(prepared_dir, target_dir, - PAGESTORE_BRANCH_BOOTSTRAP_FILE, true); + PAGESTORE_BRANCH_BOOTSTRAP_FILE, true); + pagestore_portable_install_fault(PS_FAULT_POINT_BRANCH_INSTALL_AFTER_MAPS); } /* @@ -11656,23 +11708,38 @@ pagestore_install_prepared_branch_bootstrap(PG_FUNCTION_ARGS) /* A previous successful install is no longer evidence of readiness while * maps are being replaced. Publish the new manifest only after all files. */ - snprintf(manifest_path, sizeof(manifest_path), "%s/pagestore_branch.manifest", - target_dir); - if (unlink(manifest_path) == 0) - fsync_fname(target_dir, true); - else if (errno != ENOENT) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not remove stale branch manifest \"%s\": %m", - manifest_path))); + pagestore_prepare_portable_install_faults(); + pagestore_portable_install_faults = true; + PG_TRY(); + { + snprintf(manifest_path, sizeof(manifest_path), + "%s/pagestore_branch.manifest", target_dir); + if (unlink(manifest_path) == 0) + fsync_fname(target_dir, true); + else if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove stale branch manifest \"%s\": %m", + manifest_path))); - pagestore_install_branch_bootstrap_maps(prepared_dir, target_dir, - artifact, header); - pfree(artifact); - result = DirectFunctionCall5(pagestore_install_prepared_branch, - PG_GETARG_DATUM(0), PG_GETARG_DATUM(1), - PG_GETARG_DATUM(2), PG_GETARG_DATUM(3), - PG_GETARG_DATUM(6)); + pagestore_install_branch_bootstrap_maps(prepared_dir, target_dir, + artifact, header); + pfree(artifact); + result = DirectFunctionCall5(pagestore_install_prepared_branch, + PG_GETARG_DATUM(0), PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2), PG_GETARG_DATUM(3), + PG_GETARG_DATUM(6)); + } + PG_FINALLY(); + { + pagestore_portable_install_faults = false; + if (pagestore_portable_install_faults_owned) + { + ps_fault_reset(); + pagestore_portable_install_faults_owned = false; + } + } + PG_END_TRY(); return result; } diff --git a/contrib/pagestore/pagestore_fault_points.def b/contrib/pagestore/pagestore_fault_points.def index 1478b4bd0a185..650bfe24ee876 100644 --- a/contrib/pagestore/pagestore_fault_points.def +++ b/contrib/pagestore/pagestore_fault_points.def @@ -27,3 +27,7 @@ PAGESTORE_FAULT_POINT(IMAGE_LAYER_AFTER_CREATE, "image_layer.after_create", "sto PAGESTORE_FAULT_POINT(IMAGE_LAYER_AFTER_WRITE, "image_layer.after_write", "store", "process_abort", "crash", 1, 1) PAGESTORE_FAULT_POINT(IMAGE_LAYER_AFTER_SEAL, "image_layer.after_seal", "store", "process_abort", "crash", 1, 1) PAGESTORE_FAULT_POINT(IMAGE_LAYER_AFTER_MANIFEST_ADD, "image_layer.after_manifest_add", "store", "process_abort", "crash", 1, 1) +PAGESTORE_FAULT_POINT(BRANCH_INSTALL_AFTER_MAPS, "branch_install.after_maps", "branch", "process_abort", "crash", 1, 1) +PAGESTORE_FAULT_POINT(BRANCH_INSTALL_AFTER_SLRU_REMOVE, "branch_install.after_slru_remove", "branch", "process_abort", "crash", 1, 1) +PAGESTORE_FAULT_POINT(BRANCH_INSTALL_BEFORE_MANIFEST, "branch_install.before_manifest", "branch", "process_abort", "crash", 1, 1) +PAGESTORE_FAULT_POINT(BRANCH_INSTALL_AFTER_MANIFEST, "branch_install.after_manifest", "branch", "process_abort", "crash", 1, 1)