From c3940731fe470b75247e2b2811e30c54c5aad6cd Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 3 Aug 2026 17:22:25 +0900 Subject: [PATCH 1/7] cache: shared-memory-backed Dir for fast restart Cold-start cache initialization rebuilds each stripe's in-memory directory from disk on every restart, which is multi-minute on large caches. Host the directory in POSIX shared memory so the next process start attaches the existing segment in milliseconds instead of rebuilding it. Recovery stays binary and fail-safe: when the segment cannot be trusted -- crash, reboot, ABI or schema mismatch, storage change, or failed validation -- the start drops it and rebuilds through the existing disk path. Reads still validate Doc magic and key, so a stale entry is a miss and never corruption. A stripe the previous shutdown could not vouch for is marked in the control segment, never in the stripe's own header: that header aliases raw_dir, which is also the source buffer for the on-disk directory write, so a mark there could reach disk and make the next start clear the stripe instead of recovering it. Opt-in behind proxy.config.cache.shm.enabled, default 0, where it is a functional no-op. `traffic_ctl cache shm status` and `clear` inspect and drop segments out of band. The design and the full recovery matrix are in doc/developer-guide/cache-architecture/shm-fast-restart.en.rst. --- CMakeLists.txt | 8 + doc/admin-guide/files/records.yaml.en.rst | 102 ++ .../cache-architecture/index.en.rst | 1 + .../shm-fast-restart.en.rst | 814 +++++++++++++++ include/shared/cache_shm/Layout.h | 129 +++ include/shared/cache_shm/Purge.h | 295 ++++++ include/tscore/ink_config.h.cmake.in | 1 + src/iocore/cache/AggregateWriteBuffer.cc | 1 - src/iocore/cache/AggregateWriteBuffer.h | 2 +- src/iocore/cache/CMakeLists.txt | 6 + src/iocore/cache/CacheDir.cc | 43 +- src/iocore/cache/CacheProcessor.cc | 7 + src/iocore/cache/CacheShm.cc | 932 ++++++++++++++++++ src/iocore/cache/CacheShm.h | 86 ++ src/iocore/cache/P_CacheDir.h | 14 +- src/iocore/cache/Stripe.cc | 92 +- src/iocore/cache/Stripe.h | 7 +- src/iocore/cache/StripeSM.cc | 49 +- src/iocore/cache/unit_tests/test_CacheShm.cc | 343 +++++++ .../cache/unit_tests/test_CacheShmShutdown.cc | 417 ++++++++ src/iocore/cache/unit_tests/test_Stripe.cc | 32 - src/iocore/cache/unit_tests/test_doubles.h | 35 + src/records/RecordsConfig.cc | 9 + src/traffic_ctl/CMakeLists.txt | 1 + src/traffic_ctl/CacheShmCommand.cc | 302 ++++++ src/traffic_ctl/CacheShmCommand.h | 50 + src/traffic_ctl/traffic_ctl.cc | 24 +- src/traffic_layout/info.cc | 1 + .../cache/cache_shm_bad_disk_dropped.test.py | 230 +++++ .../cache/cache_shm_concurrent_attach.test.py | 185 ++++ .../cache_shm_control_size_mismatch.test.py | 202 ++++ .../cache/cache_shm_dir_invalid.test.py | 237 +++++ .../cache/cache_shm_fast_restart.test.py | 247 +++++ .../cache/cache_shm_purge_on_disable.test.py | 232 +++++ .../cache/cache_shm_schema_mismatch.test.py | 207 ++++ .../cache/cache_shm_storage_mismatch.test.py | 210 ++++ .../cache/cache_shm_unclean_shutdown.test.py | 185 ++++ .../gold/cache_shm_state_after_shutdown.gold | 13 + .../replay/cache-shm-dir-invalid.replay.yaml | 114 +++ .../replay/cache-shm-fast-restart.replay.yaml | 87 ++ tests/gold_tests/cache/shm_poke.py | 47 + .../gold_tests/logging/ts_process_handler.py | 4 +- 42 files changed, 5930 insertions(+), 73 deletions(-) create mode 100644 doc/developer-guide/cache-architecture/shm-fast-restart.en.rst create mode 100644 include/shared/cache_shm/Layout.h create mode 100644 include/shared/cache_shm/Purge.h create mode 100644 src/iocore/cache/CacheShm.cc create mode 100644 src/iocore/cache/CacheShm.h create mode 100644 src/iocore/cache/unit_tests/test_CacheShm.cc create mode 100644 src/iocore/cache/unit_tests/test_CacheShmShutdown.cc create mode 100644 src/traffic_ctl/CacheShmCommand.cc create mode 100644 src/traffic_ctl/CacheShmCommand.h create mode 100644 tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py create mode 100644 tests/gold_tests/cache/cache_shm_concurrent_attach.test.py create mode 100644 tests/gold_tests/cache/cache_shm_control_size_mismatch.test.py create mode 100644 tests/gold_tests/cache/cache_shm_dir_invalid.test.py create mode 100644 tests/gold_tests/cache/cache_shm_fast_restart.test.py create mode 100644 tests/gold_tests/cache/cache_shm_purge_on_disable.test.py create mode 100644 tests/gold_tests/cache/cache_shm_schema_mismatch.test.py create mode 100644 tests/gold_tests/cache/cache_shm_storage_mismatch.test.py create mode 100644 tests/gold_tests/cache/cache_shm_unclean_shutdown.test.py create mode 100644 tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold create mode 100644 tests/gold_tests/cache/replay/cache-shm-dir-invalid.replay.yaml create mode 100644 tests/gold_tests/cache/replay/cache-shm-fast-restart.replay.yaml create mode 100644 tests/gold_tests/cache/shm_poke.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 520866729c6..f91b3546861 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -574,6 +574,9 @@ check_symbol_exists(mcheck_pedantic mcheck.h HAVE_MCHECK_PEDANTIC) check_symbol_exists(posix_fadvise fcntl.h HAVE_POSIX_FADVISE) check_symbol_exists(posix_fallocate fcntl.h HAVE_POSIX_FALLOCATE) check_symbol_exists(posix_madvise sys/mman.h HAVE_POSIX_MADVISE) +# On glibc < 2.34 shm_open/shm_unlink live in librt, which ATS does not link, so the cache shm feature is compiled out +# there. This is a link test, so it fails exactly on those platforms. +check_symbol_exists(shm_open sys/mman.h HAVE_SHM_OPEN) check_symbol_exists(accept4 sys/socket.h HAVE_ACCEPT4) check_symbol_exists(eventfd sys/eventfd.h HAVE_EVENTFD) check_symbol_exists(sysconf unistd.h HAVE_SYSCONF) @@ -610,6 +613,11 @@ if(HAVE_IOURING AND USE_IOURING) set(TS_USE_LINUX_IO_URING 1) endif(HAVE_IOURING AND USE_IOURING) +option(ENABLE_CACHE_SHM "Host the cache directory in POSIX shared memory for fast restart" 1) +if(HAVE_SHM_OPEN AND ENABLE_CACHE_SHM) + set(TS_USE_CACHE_SHM 1) +endif(HAVE_SHM_OPEN AND ENABLE_CACHE_SHM) + list(APPEND CMAKE_REQUIRED_LIBRARIES pthread) check_symbol_exists(pthread_getname_np pthread.h HAVE_PTHREAD_GETNAME_NP) check_symbol_exists(pthread_get_name_np pthread.h HAVE_PTHREAD_GET_NAME_NP) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index a34387bb4d3..25c4bec22ef 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -3068,6 +3068,108 @@ RAM Cache Compression runs on task threads. To use more cores for RAM cache compression, increase :ts:cv:`proxy.config.task_threads`. +.. _admin-cache-shm-fast-restart: + +Shared Memory Fast Restart +========================== + +|TS| can optionally keep the cache directory -- the in-memory index that maps +cached objects to their location on disk -- in POSIX shared memory so that it +survives a process restart. On a normal start the directory is read from disk +and, for a large cache, rebuilt in memory before the cache comes online. When +this feature is enabled and the previous instance shut down cleanly, the new +instance attaches the existing shared memory segments and skips that work, +bringing the cache online much faster. + +The shared memory directory is only an optimization for restart time; the +on-disk cache always remains the source of truth. A new instance discards the +segments and falls back to reading the directory from disk whenever they cannot +be trusted, including when: + +- the previous instance did not shut down cleanly (for example, it crashed), +- the |TS| binary's directory structures changed (an ABI mismatch, such as + after an upgrade), or +- the shared memory schema version changed. + +A change to the on-disk storage layout in :file:`storage.yaml` does *not* +discard everything. Each cache stripe attaches by its own identity, so stripes +that still exist are fast-attached, added or resized stripes are rebuilt from +disk, and segments for stripes that no longer exist are reclaimed. + +Segments left over from a crash can be inspected or removed with +``traffic_ctl cache shm status`` and ``traffic_ctl cache shm clear``, which act +directly on the shared memory objects whether or not |TS| is running. + +.. note:: + + This is an experimental feature, disabled by default. All of its settings + take effect only on a restart of |TS|. + +.. ts:cv:: CONFIG proxy.config.cache.shm.enabled INT 0 + + Enables the shared memory cache directory described above. When ``0`` (the + default), the cache directory is always read from disk on start. + + The feature requires ``shm_open()`` in libc, which |TS| checks for at build + time. Where it is absent -- glibc older than 2.34, which keeps it in + ``librt`` -- the feature is compiled out and this setting has no effect: |TS| + logs a warning at startup and every stripe uses a heap directory. That + applies to RHEL/Rocky 8, Ubuntu 20.04 and Debian 11 among others; run + ``traffic_layout info | grep TS_USE_CACHE_SHM`` to check a given build. + +.. ts:cv:: CONFIG proxy.config.cache.shm.name_prefix STRING ats + + The word used to name the POSIX shared memory objects, which on Linux appear + under ``/dev/shm``. Set only the middle word (default ``ats``); |TS| frames it + as ``/-`` so the leading ``/`` that POSIX requires and the trailing + ``-`` separator cannot be mis-typed. With the default the control segment is + named ``/ats-control`` and each per-stripe directory segment ``/ats-s`` + (for example ``/ats-s0``). Any stray framing characters are trimmed, so a + value carried over from an older release (such as ``/ats-``) still resolves to + the same names. Give each |TS| instance sharing a host a distinct word so + their segments do not collide. + + Renaming this value does not remove segments created under the old prefix: + |TS| only manages segments under the *current* prefix, so the old ``/dev/shm`` + objects linger until cleared manually with ``traffic_ctl cache shm clear + --prefix `` (or a host reboot). + +.. ts:cv:: CONFIG proxy.config.cache.shm.use_hugepages INT 0 + + When enabled (``1``), |TS| attempts to back the shared memory directory with + huge pages, which cut the cost of tearing down the large directory's page + tables at process exit (and ease TLB pressure). This requires the shared + memory to be eligible for huge pages (for example, ``/dev/shm`` mounted with + huge page support on Linux). When it is not, |TS| logs a debug message under + the ``cache_shm`` tag and transparently falls back to ordinary pages, so + enabling this is always safe. + + This advises transparent huge pages (``MADV_HUGEPAGE``) on the mapping; the + reserved ``MAP_HUGETLB`` pages used by the global hugepage allocator cannot + back a ``tmpfs``-backed shared memory segment. To avoid silently downgrading a + box that already runs with :ts:cv:`proxy.config.allocator.hugepages` enabled, + |TS| turns this on automatically when the global allocator is enabled and this + record is left at its default; set it to ``0`` explicitly to opt out. + +.. ts:cv:: CONFIG proxy.config.cache.shm.purge_stale_on_start INT 0 + + When enabled (``1``) and :ts:cv:`proxy.config.cache.shm.enabled` is ``0``, + |TS| removes any leftover shared memory segments for + :ts:cv:`proxy.config.cache.shm.name_prefix` at startup (the ``control`` + segment and the per-stripe segments it lists). This guards against two + hazards of running with the feature disabled after it had been enabled: + + - the leftover segments keep consuming memory (for example ``/dev/shm`` on + Linux) even though the disabled instance never reads them, and + - a later run with the feature re-enabled would otherwise fast-attach a + directory that went stale while |TS| ran disabled and wrote only to disk. + + The purge is skipped if a live process still owns the segments (a concurrent + instance using the same prefix), and it never blocks startup. It has no + effect when the feature is enabled, when no ``control`` segment + exists, or when set to ``0`` (the default). ``traffic_ctl cache shm clear`` + performs the same cleanup on demand. + .. _admin-heuristic-expiration: Heuristic Expiration diff --git a/doc/developer-guide/cache-architecture/index.en.rst b/doc/developer-guide/cache-architecture/index.en.rst index 4e78f8febc3..1c9da957c70 100644 --- a/doc/developer-guide/cache-architecture/index.en.rst +++ b/doc/developer-guide/cache-architecture/index.en.rst @@ -41,5 +41,6 @@ understanding and modifying the source. api-functions.en consistency.en ram-cache.en + shm-fast-restart.en cache-tool.en tiered-storage.en diff --git a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst new file mode 100644 index 00000000000..62bd47a08f1 --- /dev/null +++ b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst @@ -0,0 +1,814 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs +.. default-domain:: cpp + +.. _cache-shm-fast-restart: + +Shared-Memory Cache Directory (Fast Restart) +******************************************** + +.. note:: + + This is an experimental feature, disabled by default. It is controlled by + the ``proxy.config.cache.shm.*`` settings (see :ref:`configuration + `). The administrator-facing description lives at + :ref:`admin-cache-shm-fast-restart`; this document covers the design. + +.. note:: + + The whole feature is guarded by the ``TS_USE_CACHE_SHM`` build flag, derived + from a ``check_symbol_exists(shm_open sys/mman.h)`` probe (``HAVE_SHM_OPEN``) + and the ``ENABLE_CACHE_SHM`` option. On glibc older than 2.34 + ``shm_open()``/``shm_unlink()`` live in ``librt`` rather than libc; |TS| does + not link ``librt``, so on such platforms the flag is 0, ``CacheShm.cc`` + compiles to no-op stubs that keep ``Mode::Disabled``, and every stripe takes + the heap path exactly as it did before this feature existed. + + That covers glibc 2.17 through 2.33, which includes RHEL/Rocky 8, Ubuntu + 20.04 and Debian 11. Those builds succeed and run normally; they simply never + fast-restart. This is a deliberate decision not to carry old-platform support + for a new, opt-in feature, not an oversight -- if the feature is wanted there, + the probe needs a second pass with ``-lrt`` and the resulting library added to + the ``inkcache`` and ``traffic_ctl`` link lines. + + ``traffic_layout info`` reports ``TS_USE_CACHE_SHM``, and setting + :ts:cv:`proxy.config.cache.shm.enabled` on a build without it logs a warning + at startup rather than failing silently. + +Motivation +========== + +The :ref:`cache directory ` is the memory-resident index that +maps cached objects to their location on disk. It is rebuilt every time |TS| +starts: each stripe reads its two on-disk directory copies, picks the newer +valid one, and then runs recovery (``StripeSM::recover_data``) to replay +the fragments written since the last directory sync. For a large cache this is +the dominant cost of a restart -- the cache is not online, and therefore not +serving from cache, until it finishes. + +The directory itself, however, is purely a function of state |TS| already had +in memory in the previous process. If that memory could *survive* the process +restart, the new process could attach it and come online immediately, skipping +both the disk read and recovery. + +The shared-memory fast-restart feature does exactly that. It hosts each +stripe's ``Directory::raw_dir`` buffer in a POSIX shared-memory segment +(:manpage:`shm_open(3)`, on Linux backed by ``tmpfs`` under ``/dev/shm``). +Because the segment is owned by the kernel and not by the process, it outlives +an orderly ``traffic_server`` exit. The next start re-maps the existing segment +in milliseconds instead of rebuilding from disk. + +Design principles +================= + +The feature is built around two non-negotiable invariants. + +**The on-disk cache is always the source of truth.** The shared-memory +directory is *only* an optimization of restart time. The data fragments +themselves are never kept in shared memory -- they are read from disk on demand +exactly as before. The shared segment holds the directory index and nothing +else. + +**Recovery is binary.** The shared segment is either trustworthy enough to +attach wholesale, or it is dropped and the stripe rebuilds from disk through +the existing cold-start path. There is no attempt to repair, partially trust, +checksum, or torn-write-detect the segment. Every gate described below is a +fail-closed test: if anything is wrong or even ambiguous, the answer is "drop +and rebuild," which is always correct because the disk is authoritative. + +This keeps the trusted code small. The fast path adds no new durability +mechanism; it borrows the one the cache already has. Whenever the shared +segment is unavailable for any reason, |TS| takes precisely the path it takes +today after an unclean shutdown. + +Object layout +============= + +The feature uses two kinds of shared-memory object, defined in +:ts:git:`include/shared/cache_shm/Layout.h`. + +.. code-block:: text + + POSIX shared memory (e.g. /dev/shm on Linux) + + control one per traffic_server instance + +-------------------------------------------------------------+ + | magic "ATS-SHM\0" schema_version abi_hash | + | storage_signature clean_shutdown owner_pid | + | stripe_count | + | stripes[0 .. MAX_STRIPES-1]: | + | { shm_name, raw_dir_size, stripe_key_hash, | + | dir_untrusted } | + +-------------------------------------------------------------+ + | | | + v v v + s0 s1 s2 per-stripe raw_dir + +-----------+ +-----------+ +-----------+ + | header | | header | | header | StripeHeaderFooter + | dir[] | | dir[] | | dir[] | directory entries + | footer | | footer | | footer | + +-----------+ +-----------+ +-----------+ + +The control segment +------------------- + +There is one control segment per instance, named ``control``. It is a +fixed-size ``cache_shm::CacheShmControl`` -- a header plus a table of +up to ``MAX_STRIPES`` (256) ``cache_shm::StripeEntry`` rows. A +``static_assert`` keeps the whole control segment under 32 KiB. Its fields: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Field + - Purpose + * - ``magic`` + - ``"ATS-SHM\0"``. Identifies a |TS| control segment and is the first + thing checked on attach. + * - ``schema_version`` + - The on-shm wire-format version. Bumped whenever the meaning of the + layout changes; a mismatch drops the segment. + * - ``abi_hash`` + - A compile-time fingerprint of the binary's directory structures (see + ``CacheShm::abi_hash``). A mismatch -- e.g. after an upgrade that + changed ``Dir`` -- drops the segment. + * - ``storage_signature`` + - A fingerprint of the ``storage.yaml`` topology. **Not** a hard + gate; see `Storage changes and partial attach`_. + * - ``clean_shutdown`` + - ``1`` only between a clean shutdown and the next attach. ``0`` at all + other times, including throughout a running process, so a crash leaves + it ``0``. + * - ``owner_pid`` + - PID of the process currently mapping the segment read-write, or ``0`` + when none. Backs the concurrent-attach guard. + * - ``stripe_count`` + - High-water mark of used rows in ``stripes[]``. + * - ``stripes[]`` + - One row per stripe: its segment name, the segment's byte size, the + 64-bit stripe identity hash used to match a stripe to its prior segment, + and ``dir_untrusted`` (see :ref:`shm-marking-untrusted`). + +Per-stripe directory segments +----------------------------- + +Each stripe's directory lives in its own segment, ``s``. The mapped +region *is* the stripe's ``Directory::raw_dir``: the +:cpp:class:`StripeHeaderFooter` header, the array of :cpp:class:`Dir` entries, +and the footer, in exactly the same byte layout the cache writes to disk. A +stripe reads and writes its directory through this mapping for the entire run, +so the segment is continuously current -- there is no separate "flush to shared +memory" step. + +Naming +------ + +All names derive from :ts:cv:`proxy.config.cache.shm.name_prefix`, which is just +the middle word (default ``ats``). |TS| frames that word as ``/-`` -- the +leading ``/`` that POSIX shared memory requires and the trailing ``-`` separator +are supplied by ``cache_shm::normalize_name_prefix``, not the operator, +so neither can be mis-typed; any stray framing carried over from an older config +(for example a literal ``/ats-``) is trimmed first, so it can never become an +invalid embedded-slash name like ``//ats--``. With the default word the framed +prefix is ``/ats-``: the control segment is ``/ats-control`` and stripe segments +are ``/ats-s`` where ``N`` is a per-instance slot index. Names are kept under +``cache_shm::MAX_SHM_NAME_LEN`` (31) characters because macOS caps POSIX +shared-memory names (``PSHMNAMLEN``) at 31 including the leading ``/``; keeping +to that limit makes the same naming work on Linux and macOS. Instances sharing +a host **must** use distinct words so their segments do not collide. + +Note that the stripe segment name is just a slot label. A stripe is matched to +its prior segment by ``stripe_key_hash`` (a 64-bit FNV-1a of the stripe's +``hash_text``), **not** by name or index, so a span going offline can shift +slot numbers without breaking the identity match. + +Startup +======= + +``CacheShm::initialize`` runs from +``CacheProcessor::start_internal``, after the :cpp:class:`Store` is +read but before any :cpp:class:`Stripe` is constructed. It loads the +configuration, then opens the control segment and selects one of three modes: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Mode + - Meaning + * - ``Disabled`` + - The feature is off (or a fatal precondition failed, such as a name that + is too long or losing the concurrent-attach race). Stripes use the + normal heap/hugepage directory; behavior is identical to stock |TS|. + * - ``AttachExisting`` + - A trustworthy prior control segment exists. Stripes attach their prior + segment by identity, or create a fresh one where there is no match. + * - ``CreateFresh`` + - No usable prior control segment. A new one is created and every stripe + segment is created empty (the cold path, but now shared-memory-backed + for *next* time). + +Trust gates +----------- + +When a prior control segment exists, ``initialize`` applies these gates in +order. The first failure drops the entire control segment (unlinking every +stripe segment it lists) and falls through to ``CreateFresh``: + +.. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Gate + - Drops the segment when... + * - concurrent-attach guard + - another live process is mapping the segment (see below). This actually + disables shared memory for the run rather than dropping -- the live + owner's segment must be left intact. + * - segment size + - the segment is not ``sizeof(CacheShmControl)`` bytes (rounded up to a + page), i.e. it was written by a binary with a different control layout. + Only the frozen header is mapped in that case; the stripe table behind it + cannot be interpreted, so the whole ``s`` name space is + unlinked by name instead of being read from the table. + * - ``magic`` + - the magic bytes do not match (not our segment, or corrupt). + * - ``schema_version`` + - the on-shm format version differs from this binary's. + * - ``abi_hash`` + - the binary's directory structures differ from the writer's (e.g. an + upgrade changed ``Dir``, ``StripeHeaderFooter``, ``DIR_DEPTH``, ...). + * - ``clean_shutdown`` + - the previous run did not set it to ``1`` -- i.e. it crashed or was + killed. A crash may have left directory entries pointing at fragments + that were never flushed, so no stripe can safely skip recovery. + +If every gate passes, ``initialize`` adopts the segment: it records itself as +``owner_pid``, sets ``clean_shutdown = 0`` (so a crash *this* run drops the +segment next time), ``msync``\ s the header, and enters ``AttachExisting``. The +per-stripe work then happens lazily as each stripe initializes. + +The frozen control header +------------------------- + +The gates above have to work on a segment this binary did not write, including +one whose ``sizeof(CacheShmControl)`` differs -- a ``MAX_STRIPES`` bump, a +longer ``shm_name``, a new ``StripeEntry`` field. Everything the gates read +therefore lives in a **frozen prefix** of the struct, the bytes ahead of +``stripes[]`` (``cache_shm::CONTROL_HEADER_SIZE``, pinned by a +``static_assert``): ``magic``, ``schema_version``, ``abi_hash``, +``storage_signature``, ``clean_shutdown``, ``owner_pid`` and ``stripe_count``. +Append to ``StripeEntry`` or grow ``stripes[]`` freely; never reorder or extend +that prefix. + +This is what makes a layout change survivable. Without it, the size-checked +attach would simply fail, ``initialize`` would never reach the ``abi_hash`` +gate (which needs a successful map), and the ``O_EXCL`` create that follows +would then fail with ``EEXIST`` on every restart -- shared memory silently off +until an operator ran ``traffic_ctl cache shm clear``. With it, the segment is +identified, guarded by the concurrent-attach guard, dropped, and recreated in a +single start. + +Concurrent-attach guard +----------------------- + +Two ``traffic_server`` processes must never map the same directory read-write; +the second would corrupt the first's live index. ``clean_shutdown`` is no help +here -- it says nothing about a process that is *currently* running. The guard +is therefore based on ownership, with two layers: + +* **flock.** ``initialize`` takes a non-blocking exclusive ``flock`` on the + control-segment fd and holds it for the entire process lifetime + (``g_control_fd``). The kernel releases it automatically on exit *or crash*, + so it is self-healing. If the lock is already held + (``LockResult::HeldByOther``), a live owner exists and the new process + disables shared memory for its run. This is authoritative on Linux/``tmpfs``. + +* **owner_pid liveness.** macOS POSIX shared memory does not honor ``flock`` + (``LockResult::Unsupported``). There, the guard falls back to the recorded + ``owner_pid``: if it names a live process other than ourselves + (``CacheShm::process_is_alive``, via ``kill(pid, 0)``), the new + process disables shared memory. A clean shutdown clears ``owner_pid`` to + ``0``; a crash leaves a stale pid, but a crash also leaves + ``clean_shutdown = 0``, so the segment is dropped by that gate anyway. + +A symmetric check guards the ``CreateFresh`` path: after creating the fresh +control segment, ``initialize`` takes the lock, and if it lost a creation race +to another starting process it backs out and disables shared memory for the +run. + +Per-stripe attach and the fast path +==================================== + +For each stripe, ``Stripe::_init_directory`` asks +``CacheShm::attach_or_create_stripe`` for its ``raw_dir`` *before* +falling back to the hugepage / aligned-heap allocation: + +.. code-block:: cpp + + this->directory.raw_dir = CacheShm::attach_or_create_stripe(hash_text.get(), directory_size); + if (this->directory.raw_dir == nullptr) { + // shm disabled or attach/create failed -> hugepage, then aligned heap + } + +``attach_or_create_stripe`` looks up the stripe by ``stripe_key_hash`` in the +control table: + +* **Match found** (and the recorded size matches): map the existing segment and + return it. This is the segment the previous run left behind. +* **No match**: reserve a fresh table slot and create a new, zero-filled + segment. + +A freshly created segment has a zero header magic, so the fast-attach gate +below rejects it and ``StripeSM::init`` falls through to the normal disk +read, which repopulates the directory in place. + +The fast-attach gate +-------------------- + +In ``AttachExisting`` mode, when ``raw_dir`` came from shared memory, +``StripeSM::init`` checks whether the in-segment directory can be trusted +without reading disk: + +#. ``header->magic`` and ``footer->magic`` are both ``STRIPE_MAGIC``; +#. the directory version is within + ``[CACHE_DB_MAJOR_VERSION_COMPATIBLE, CACHE_DB_MAJOR_VERSION]``; +#. ``Stripe::_shm_directory_is_valid`` passes (see below). + +When all three hold, the stripe skips both the disk read **and** +``StripeSM::recover_data`` -- which would otherwise rescan the tail and +discard the very entries the shared segment preserved -- and jumps straight to +the post-recovery state (``sector_size``, ``scan_pos``, +``periodic_scan``, then ``StripeSM::dir_init_done``), +mirroring the tail of ``handle_recover_write_dir()``. It logs:: + + attaching cached directory from shm for '' (fast restart, recovery skipped) + +If any check fails, it logs ``shm directory invalid ...; falling back to disk +read`` and proceeds exactly as a cold start would. + +Validating a trusted segment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The magic/version checks confirm the segment *looks like* a directory, but a +stale-yet-magic-valid segment could still present offsets that would turn into +out-of-bounds disk I/O. And ``CacheShm::mark_clean_shutdown`` runs while the +event threads are still live (see `Shutdown`_), so a directory mutation torn at +process exit can sit behind a ``clean_shutdown = 1`` flag. +``Stripe::_shm_directory_is_valid`` therefore validates both the header fields +and the shape of the entry graph before the attach: + +* ``sector_size`` is non-zero and no larger than ``STORE_BLOCK_SIZE``; +* ``write_pos``, ``last_write_pos`` and ``agg_pos`` all lie within the stripe's + data region (``[start, skip + len]``); +* ``agg_pos == write_pos``, i.e. the write cursor is quiesced. A clean shutdown + guarantees this: the shutdown path flushes the aggregation buffer (which leaves + the two equal, as ``aggWriteDone`` does) and invalidates the segment outright if + a write is still in flight. A *failed* flush leaves ``agg_pos`` past + ``write_pos``, so this same check is what drops the segment in that case -- see + `Flush failure at shutdown`_. ``last_write_pos`` is deliberately *not* compared + against ``write_pos`` -- ``agg_wrap()`` resets ``write_pos`` to ``start`` + without touching it, so ``last_write_pos > write_pos`` is a legitimate + post-wrap state; +* every per-segment free-list head indexes a ``Dir`` entry within its segment + (walking a free list from an out-of-range head would run off the end); +* every ``Dir`` entry's ``next`` indexes a ``Dir`` within the same segment. + ``dir_from_offset()`` does no bounds checking, and both bucket chains and the + free list are followed -- and written through -- all over the cache, so this has + to hold before anything follows a link. It is also what makes the walk below + safe to run; +* ``prev`` is bounds-checked only on *empty* entries. ``dir_prev`` aliases the + word that carries ``tag``/``phase``/``head``/``pinned`` on an in-use entry (see + ``dir_tag`` in :ts:git:`src/iocore/cache/P_CacheDir.h`) and is a link only on + free-list members, which are empty. Checking it on an in-use entry compares flag + bits against an entry count, which rejects healthy directories whose segments + hold fewer entries than those bits can spell -- the ``head`` bit alone is 8192, + so every stripe under roughly 65 MB failed validation and silently gave up fast + restart; +* ``Directory::check_segment()`` passes for the segment: bucket chain lengths, no + chained-but-empty entries, no chain loops. This is the ``CHECK_DIR`` walk that is + otherwise debug-only, and it is what turns a torn directory into a rebuild rather + than an attach. + +A failure here is treated like any other attach miss: drop to the disk read and +recover. + +All of the per-entry work is per-segment, so it runs as a single fused pass: +bounds-check segment *s*, walk its entry graph, move on. Bounds-checking every +segment first and only then walking them would stream the whole directory twice, +and on a multi-TB span that is two passes over a gigabyte or more from DRAM both +times -- on the path whose entire purpose is restart latency. A segment is at most +a few hundred KiB, so fusing keeps the structural walk in cache. Ordering is +unaffected because ``next_dir()`` never leaves its own segment. This is why +``Directory::check()`` is factored into a per-segment ``check_segment()``; the +whole-directory entry point keeps its previous behaviour for its other callers. + + +Storage changes and partial attach +=================================== + +A change to ``storage.yaml`` does **not** invalidate the whole control +segment. ``storage_signature`` is recorded and used only to phrase the startup +log line ("partial -- storage changed"); it is not a trust gate. The actual +reconciliation is per stripe, driven by identity: + +* A stripe whose ``stripe_key_hash`` still matches a table entry of the right + size attaches its prior segment as usual. +* A stripe that is new, relocated, or resized finds no match and creates a + fresh segment (then loads from disk). +* A table entry that *no* stripe claimed this run is an **orphan** -- its stripe + left the cache (a span was dropped, or a disk failed to open). + +``CacheShm::finalize_attach``, called from +``CacheProcessor::cacheInitialized`` once every stripe has initialized, +reclaims the orphans: it unlinks each unclaimed segment, tombstones its slot for +reuse, and trims trailing tombstones so ``stripe_count`` tracks the live +high-water mark. + +One guard matters here: if **zero** stripes claimed a segment this run, +``finalize_attach`` leaves every entry intact. Zero claims cannot be +distinguished from an aborted init (for example a transient ``volume.config`` +error), and reclaiming a valid cache's segments would be far worse than leaking +them for one run. + +Shutdown +======== + +A clean shutdown is what makes the next start fast, so the directory must be +made final and the segment marked clean -- in that order. + +Wiring +------ + +On a clean exit, ``AutoStopCont::mainEvent`` calls +``sync_cache_dir_on_shutdown()`` whenever the cache is initialized. +``sync_cache_dir_on_shutdown`` snapshots every stripe (taking each stripe mutex, +which excludes writers *concurrent with* the snapshot), and only then calls +``CacheShm::mark_clean_shutdown``, which sets ``clean_shutdown = 1``, clears +``owner_pid`` to ``0``, and ``msync``\ s the header. When the feature is +disabled, ``mark_clean_shutdown`` is a no-op (there is no control segment), so +the shutdown path is unchanged for a stock |TS|. + +It does not stop *future* writers: this runs before ``shut_down_event_system()`` +and the main thread exits without joining the event threads, so a stripe can +still be written after its snapshot. That is why the fast-attach path re-validates +the directory structurally rather than trusting ``clean_shutdown`` alone -- see +`Validating a trusted segment`_. A late write that only lands a directory entry +is harmless either way: the read path checks ``Doc`` magic and key before serving, +so a stale entry resolves to a miss. + +Skipping the on-disk directory write +------------------------------------ + +For a shared-memory-backed stripe, ``StripeSM::shutdown`` still flushes +the aggregation buffer (so pending *content* reaches disk) but then **skips the +on-disk directory write** entirely: + +.. code-block:: text + + Dir : shm-backed, skipping on-disk directory write + +The shared segment is already the durable copy of the directory and is attached +directly next start, so writing the A/B copies to disk would be pure waste. The +trade-off is deliberate and safe: if the segment is later dropped for any +reason, the on-disk A/B copies plus ``StripeSM::recover_data`` reconcile +the tail -- the same path an unclean restart already takes. + +Flush failure at shutdown +------------------------- + +If the aggregation-buffer flush at shutdown fails (e.g. the disk went bad), the +on-disk content no longer matches the directory, so the shared segment must not +be trusted next start. The stripe is marked (see :ref:`shm-marking-untrusted`) +rather than left to the quiesced-cursor check: the failure does leave ``agg_pos`` +ahead of ``write_pos``, but the event system is still up, so a later +``aggWriteDone`` or ``agg_wrap()`` can re-equalize the two and the gate would +then let the segment through. + +``StripeSM::shutdown`` then still writes the on-disk directory for that stripe, +exactly as it does without this feature. That directory may reference content the +failed flush never wrote, but such entries fail the ``Doc`` magic and key check +on read and register as a miss; skipping the write would instead discard every +directory insert since the last periodic sync. + +.. _shm-marking-untrusted: + +Marking a stripe untrusted +-------------------------- + +Two shutdown paths cannot vouch for the directory at all: the disk was already +marked bad, and an aggregation write is still in flight (``aggWriteDone`` will +advance ``write_pos`` again once the mutex is dropped, so the in-segment header +is not final). Both call ``CacheShm::invalidate_stripe_directory``, which sets +``dir_untrusted`` on that stripe's ``StripeEntry`` in the **control** segment and +``msync``\ s it. + +The mark deliberately does not live in the stripe's own header. That header +aliases ``raw_dir``, and ``raw_dir`` is simultaneously the live directory, the +segment that persists across restarts, *and* the source buffer that both the +shutdown ``pwrite`` and the periodic dir sync (``CacheDir.cc``'s +``memcpy(buf, raw_dir, dirlen)``) copy to disk. A mark written there can reach +the on-disk A/B copy, and a zeroed magic on disk makes the next start *clear* +the stripe rather than recover it. The control segment is never a disk-write +source, so it cannot leak that way. + +Next start, ``attach_or_create_stripe`` finds the entry by identity as usual, +sees the mark, and creates a fresh segment instead of attaching; the fresh +segment's zero magic sends ``StripeSM::init`` down the ordinary disk-read and +``recover_data`` path. The marked entry is tombstoned and its segment unlinked at +that moment, rather than left as an orphan for ``finalize_attach``: the slot is +reused immediately, so ``stripe_count`` cannot creep across runs that never reach +``finalize_attach`` (an init aborted before ``cacheInitialized()`` skips it), and +the old and new segments are never held at once. Between the shutdown that set +the mark and that next start, ``traffic_ctl cache shm status`` shows the row as +``untrusted``. + +Crash and recovery summary +========================== + +The state machine reduces to: *the segment is attached only when it is provably +consistent, and dropped otherwise.* + +.. list-table:: + :header-rows: 1 + :widths: 34 66 + + * - Event between runs + - Next start + * - Clean shutdown, unchanged binary & storage + - Fast attach. Recovery skipped. Cache online in milliseconds. + * - Crash / ``SIGKILL`` + - ``clean_shutdown`` still ``0`` -> drop, rebuild from disk + recover. + * - Binary upgrade changing directory structures + - ``abi_hash`` mismatch -> drop, rebuild. + * - Binary upgrade changing the control layout + - Segment size differs -> drop (stripe segments unlinked by name), recreate, + rebuild. Never wedges the create path; see `The frozen control header`_. + * - Schema bump + - ``schema_version`` mismatch -> drop, rebuild. + * - Directory torn at process exit + - ``Stripe::_shm_directory_is_valid`` rejects that stripe -> it rebuilds + from disk + recover; others fast-attach. + * - ``storage.yaml`` change + - Control segment kept; matching stripes fast-attach, changed stripes + rebuild, orphans reclaimed. + * - Per-stripe shutdown flush failed + - ``agg_pos != write_pos`` -> that stripe rebuilds from its freshly written + on-disk directory; others fast-attach. + * - Bad disk, or an AIO write still in flight at shutdown + - ``dir_untrusted`` set on that stripe's control entry -> it is recreated and + rebuilds from disk; others fast-attach. See :ref:`shm-marking-untrusted`. + * - Another live owner using the prefix + - Refuse to attach; shared memory disabled for this run. + +In every "drop/rebuild" row, |TS| behaves exactly as it does today without the +feature -- the fast path is the only thing lost. + +Huge pages +========== + +The large directory segments make page-table teardown at process exit +non-trivial: ``exit_mmap`` walks O(number of PTEs), which for multi-gigabyte +directories can cost seconds. Backing the mapping with huge pages cuts the PTE +count ~512x and the teardown cost with it. + +When :ts:cv:`proxy.config.cache.shm.use_hugepages` is set, |TS| advises +transparent huge pages on the mapping with ``madvise(MADV_HUGEPAGE)``. +``MAP_HUGETLB`` is deliberately **not** used: ``shm_open`` fds are ``tmpfs`` +backed, and ``MAP_HUGETLB`` requires a ``hugetlbfs`` fd, so it always fails with +``EINVAL``. The advice requires shmem THP to be enabled on the host (for +example ``/sys/kernel/mm/transparent_hugepage/shmem_enabled`` set to ``advise`` +or ``always``, or the ``tmpfs`` mounted with ``huge=advise``). When huge pages +are unavailable the ``madvise`` simply logs a debug line under the +``cache_shm`` tag and the kernel uses base pages, so enabling the setting is +always safe. + +Because a stock directory allocation uses reserved ``MAP_HUGETLB`` pages when +:ts:cv:`proxy.config.allocator.hugepages` is enabled, a shm-backed directory +would otherwise silently drop such a box to base pages -- the shm segment cannot +use ``MAP_HUGETLB`` at all. To avoid that, |TS| turns ``shm.use_hugepages`` on +automatically when the global allocator is enabled and the record is left at its +default (advising THP as the closest substitute), and logs the substitution once +at startup. Setting ``shm.use_hugepages`` to ``0`` explicitly opts out and is +honored; the opt-out is logged as a warning so the base-page choice is visible. + +Concurrency model +================= + +Stripes initialize concurrently across the AIO/disk threads, so the +control-table bookkeeping is locked, but the slow shared-memory syscalls are +kept out of the critical section: + +* ``g_table_mutex`` guards the control-segment stripe table and the per-run + claim bookkeeping. ``attach_or_create_stripe`` decides what to do (reuse a + table slot or reserve a fresh one) under the lock, then **drops it** before + ``shm_open`` / ``ftruncate`` / ``mmap``. Each stripe owns a distinct segment, + so the syscalls never touch another thread's segment. Holding the lock across + them would serialize every disk thread's init. +* ``g_pointers_mutex`` guards the set of pointers handed out, so + ``CacheShm::is_shm_pointer`` (used to tell a shm-backed directory from + a heap-allocated one, e.g. to skip the redundant on-disk directory write) is + thread-safe. +* Slot reservation tombstones a slot if the create later fails + (``release_reserved_slot``), so a failed create cannot strand a half-built + table entry. + +Disabling the feature: stale-segment purge +========================================== + +Running with the feature **disabled** after it had been enabled is hazardous in +two ways: the leftover segments keep consuming memory the disabled instance +never reads, and a later re-enabled run could fast-attach a directory that went +stale while |TS| ran disabled (writing only to disk). To address this, +:ts:cv:`proxy.config.cache.shm.purge_stale_on_start` (opt-in) makes a disabled +start best-effort remove any leftover segments for the configured prefix. + +The purge shares one primitive with the operator tooling (see below): +``cache_shm::purge_segments`` in :ts:git:`include/shared/cache_shm/Purge.h`. It +enumerates the stripe table and unlinks every stripe segment plus the control +object, returning a structured ``PurgeReport`` that each caller renders in its +own format. It refuses to unlink anything owned by a live process (the same +flock + ``owner_pid`` guard used at attach), and it never blocks startup. An +already-gone segment (``ENOENT``) is the desired end state and is not counted as +a failure. + +Operator tooling: ``traffic_ctl cache shm`` +============================================ + +Because crash-leftover segments may need inspecting when no live process is +around to query, the tooling acts on the shared-memory objects **directly**, via +``shm_open``, rather than over JSON-RPC. For that reason ``traffic_ctl`` does +**not** link the cache library; the small amount of shared logic lives in +header-only form (:ts:git:`include/shared/cache_shm/Layout.h` and +``shared/cache_shm/Purge.h``). + +``traffic_ctl cache shm status [--prefix P]`` + Maps the control segment read-only and prints its header (magic, + schema/abi/storage fingerprints, ``clean_shutdown``, and whether + ``owner_pid`` names a live process) followed by the stripe table, flagging + each segment ``present`` / ``MISSING`` and each free slot as a tombstone. + +``traffic_ctl cache shm clear [--prefix P]`` + Removes the segments via the shared ``purge_segments`` primitive. It + **refuses** to clear segments owned by a live ``traffic_server`` (stop it + first), so it cannot orphan a running instance's fast restart. This is the + on-demand equivalent of ``purge_stale_on_start``. + +.. _cache-shm-configuration: + +Configuration +============= + +All settings are under ``proxy.config.cache.shm`` and take effect only on a +restart (``RECU_RESTART_TS``). See :ref:`admin-cache-shm-fast-restart` for the +full administrator-facing descriptions. + +.. list-table:: + :header-rows: 1 + :widths: 38 12 50 + + * - Setting + - Default + - Effect + * - :ts:cv:`proxy.config.cache.shm.enabled` + - ``0`` + - Master switch. ``0`` = always read the directory from disk (stock + behavior). + * - :ts:cv:`proxy.config.cache.shm.name_prefix` + - ``ats`` + - Middle word of the shared-memory object names; framed as ``/-`` + (the ``/`` and ``-`` are added by |TS|). Give co-located instances + distinct words. + * - :ts:cv:`proxy.config.cache.shm.use_hugepages` + - ``0`` + - Advise transparent huge pages on the directory mappings. Safe when + unavailable; falls back to base pages. + * - :ts:cv:`proxy.config.cache.shm.purge_stale_on_start` + - ``0`` + - When the feature is disabled, best-effort remove leftover segments for + the prefix at startup. + +Platform considerations +======================= + +* **Linux** is the primary target: ``tmpfs`` (``/dev/shm``) backs the segments, + ``flock`` is authoritative for the concurrent-attach guard, and shmem THP + provides the huge-page teardown win. +* **macOS** is supported for development and testing on a best-effort basis. + POSIX shared-memory names are limited to 31 characters (the reason for + ``MAX_SHM_NAME_LEN``), ``flock`` is not honored on shm fds, so the + concurrent-attach guard is best-effort there: it relies on the ``owner_pid`` + liveness backstop alone (the ``kill(pid, 0)`` check), which closes the window + but cannot make the attach atomic the way ``flock`` does on Linux. The kernel + also rounds a segment up to a page boundary, so ``open_and_map_shm`` accepts + any size in ``[requested, page-up]``. +* The feature is inert at the default :ts:cv:`proxy.config.cache.shm.enabled` + ``0``: no segments are created or attached on any platform, and behavior is + identical to stock |TS|. +* Realistic multi-gigabyte directory sizes, the ``MADV_HUGEPAGE`` teardown win, + and the restart-time benchmarks are Linux-only -- the same platform boundary + |TS| already has for its hugepage directory allocation. (Recall ``MAP_HUGETLB`` + is never used here; see `Huge pages`_.) + +Testing +======= + +The pure trust-gate logic is unit-tested in +:ts:git:`src/iocore/cache/unit_tests/test_CacheShm.cc` (ABI-hash stability, the +storage-signature topology sensitivity, control-header round-trip, the macOS +name-length limit, and the process-liveness check). + +The end-to-end behavior is covered by autests in +:ts:git:`tests/gold_tests/cache/`, one scenario each: + +.. list-table:: + :header-rows: 1 + :widths: 42 58 + + * - Test + - Scenario + * - ``cache_shm_fast_restart`` + - Directory survives a clean shutdown and is fast-attached. + * - ``cache_shm_unclean_shutdown`` + - ``SIGKILL`` leaves the segment dirty; next start drops and rebuilds. + * - ``cache_shm_schema_mismatch`` + - A poked ``schema_version`` is dropped, never attached. + * - ``cache_shm_control_size_mismatch`` + - A control segment of another build's size is dropped and recreated in one + start; the next start fast-attaches what it created. + * - ``cache_shm_dir_invalid`` + - A poked in-shm stripe header (``write_pos`` past the stripe, + ``freelist[0]`` past the segment) is rejected; the stripe rebuilds from + disk and still serves a hit. + * - ``cache_shm_storage_mismatch`` + - A changed storage layout keeps the control segment, creates a fresh + relocated stripe, and reclaims the orphan. + * - ``cache_shm_bad_disk_dropped`` + - Dropping a disk fast-attaches the survivors and reclaims the removed + disk's segment. + * - ``cache_shm_concurrent_attach`` + - A second ``traffic_server`` refuses to attach over a live owner and runs + with shared memory disabled. + * - ``cache_shm_purge_on_disable`` + - ``purge_stale_on_start`` removes leftover segments on a disabled start. + +The schema, control-size, storage and directory tests drive their gates by +editing ``/dev/shm`` directly (``shm_poke.py``), which is a Linux facility; they +have no macOS condition. + +Limitations and non-goals +========================= + +* The feature accelerates restart only; it does not change steady-state cache + behavior, durability, or the on-disk format. +* Only the directory is shared, never cached content. +* There is no migration or repair of an untrusted segment -- the disk is + authoritative and rebuilding from it is always the fallback. +* A single host may run multiple instances only with distinct + ``name_prefix`` values. + +Source map +========== + +.. list-table:: + :header-rows: 1 + :widths: 42 58 + + * - File + - Role + * - :ts:git:`src/iocore/cache/CacheShm.h` / ``CacheShm.cc`` + - The ``CacheShm`` facade: initialize, attach/create, finalize, mark-clean, + invalidate, and the trust-gate fingerprints. + * - :ts:git:`include/shared/cache_shm/Layout.h` + - The on-shm control-segment layout, shared with tooling. + * - :ts:git:`include/shared/cache_shm/Purge.h` + - The header-only enumerate-and-unlink primitive and its owner guard, + shared by the disabled-start purge and ``traffic_ctl``. + * - :ts:git:`src/iocore/cache/Stripe.cc` + - The shared-memory ``raw_dir`` allocation and ``_shm_directory_is_valid``. + * - :ts:git:`src/iocore/cache/StripeSM.cc` + - The fast-attach gate in ``StripeSM::init`` and the shutdown-write skip / + untrusted mark in ``StripeSM::shutdown``. + * - :ts:git:`src/iocore/cache/CacheProcessor.cc` + - ``initialize`` / ``finalize_attach`` call sites in ``CacheProcessor``. + * - :ts:git:`src/iocore/cache/CacheDir.cc` + - ``mark_clean_shutdown`` from ``sync_cache_dir_on_shutdown``. + * - :ts:git:`src/traffic_ctl/CacheShmCommand.cc` + - The ``traffic_ctl cache shm status`` / ``clear`` commands. diff --git a/include/shared/cache_shm/Layout.h b/include/shared/cache_shm/Layout.h new file mode 100644 index 00000000000..bda3a8539d4 --- /dev/null +++ b/include/shared/cache_shm/Layout.h @@ -0,0 +1,129 @@ +/** @file + + Layout of the cache shared-memory control segment, shared between the cache + subsystem and tools (traffic_ctl) that inspect or clear the segment without + going through the running traffic_server. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "tscore/ink_align.h" +#include "tscore/ink_memory.h" + +#include +#include +#include +#include +#include + +namespace cache_shm +{ + +constexpr char CACHE_SHM_MAGIC[8] = {'A', 'T', 'S', '-', 'S', 'H', 'M', '\0'}; +constexpr uint32_t CACHE_SHM_SCHEMA_VERSION = 1; +constexpr std::string_view CACHE_SHM_CONTROL = "control"; + +// macOS PSHMNAMLEN is 31 chars including the leading '/'. Keep names under that +// limit on Linux too, so the same naming works everywhere. +constexpr std::size_t MAX_SHM_NAME_LEN = 31; + +// Maximum number of stripes in the control segment. Bumping it changes both the ABI +// hash and sizeof(CacheShmControl); a prior segment is dropped on attach either way +// (see CONTROL_HEADER_SIZE for how the size change is detected). +constexpr std::size_t MAX_STRIPES = 256; + +// Per-stripe entry in the control segment. A stripe is matched to its prior +// segment on attach by stripe_key_hash, not by name (order-independent). +// dir_untrusted lives here and not in the stripe's own header because that header aliases raw_dir, which is also the source +// buffer for the on-disk directory write -- a mark written there can reach disk and cost the stripe on the next start. +struct StripeEntry { + char shm_name[MAX_SHM_NAME_LEN + 1]; ///< full shm name, NUL-terminated. + uint64_t raw_dir_size; ///< size of the stripe's raw_dir segment, bytes. + uint64_t stripe_key_hash; ///< full 64-bit FNV-1a of the stripe hash_text. + uint8_t dir_untrusted; ///< 1 = shutdown could not vouch for this directory; never attach it again. + uint8_t pad0[7]; +}; + +struct CacheShmControl { + char magic[8]; ///< CACHE_SHM_MAGIC + uint32_t schema_version; ///< CACHE_SHM_SCHEMA_VERSION + uint32_t pad0; + uint64_t abi_hash; ///< compile-time ABI fingerprint + uint64_t storage_signature; ///< storage.yaml fingerprint + uint8_t clean_shutdown; ///< 0 = dirty, 1 = clean + uint8_t pad1[3]; + int32_t owner_pid; ///< PID holding the exclusive lock; 0 when none. Backs the + ///< concurrent-attach guard. Cleared on clean shutdown. + uint32_t stripe_count; + uint32_t pad2; + StripeEntry stripes[MAX_STRIPES]; +}; + +constexpr std::size_t CONTROL_SIZE = sizeof(CacheShmControl); + +// FROZEN: append to StripeEntry or grow stripes[] freely, but never reorder or extend the bytes ahead of stripes[]. A +// build with a different sizeof(CacheShmControl) must still be able to read this far to drop the segment rather than +// wedge on EEXIST forever. See "The frozen control header" in the shm-fast-restart developer guide. +constexpr std::size_t CONTROL_HEADER_SIZE = offsetof(CacheShmControl, stripes); +static_assert(CONTROL_HEADER_SIZE == 48, "the control segment header is a frozen layout; see the comment above"); +static_assert(std::is_standard_layout_v, "the control segment is shared across processes and builds"); + +// Whether a control segment of `actual` bytes was written by *this* build; the kernel rounds an shm object up to a page. +// Anything larger has a stripes[] of unknown stride and must never be walked with our layout. Shared by the attach gate, +// the purge primitive and `traffic_ctl cache shm status` so the three cannot drift apart. +inline bool +is_own_control_size(std::size_t actual) +{ + return actual >= CONTROL_SIZE && actual <= INK_ALIGN(CONTROL_SIZE, ats_pagesize()); +} + +// Frame the operator's middle word (e.g. "ats") as "/-". The framing is supplied here so it cannot be mis-typed: +// stray framing from an older config is trimmed, and embedded '/' stripped since POSIX permits only the leading one. +inline std::string +normalize_name_prefix(std::string_view configured) +{ + std::size_t begin = configured.find_first_not_of('/'); + if (begin == std::string_view::npos) { + begin = configured.size(); // all '/' (or empty): no middle. + } + std::size_t last_kept = configured.find_last_not_of('-'); + std::string_view middle = (last_kept == std::string_view::npos || last_kept < begin) ? + std::string_view{} : + configured.substr(begin, last_kept - begin + 1); + std::string word{"/"}; + for (char c : middle) { + if (c != '/') { // POSIX shm names allow only the leading '/'. + word += c; + } + } + word += "-"; + return word; +} + +// Name of the "control" segment. Derived here so the cache subsystem and +// traffic_ctl agree; `prefix` is the normalized prefix (e.g. "/ats-"). +inline std::string +control_segment_name(std::string_view prefix) +{ + return std::string(prefix) + CACHE_SHM_CONTROL.data(); +} + +} // namespace cache_shm diff --git a/include/shared/cache_shm/Purge.h b/include/shared/cache_shm/Purge.h new file mode 100644 index 00000000000..f48ffa908e7 --- /dev/null +++ b/include/shared/cache_shm/Purge.h @@ -0,0 +1,295 @@ +/** @file + + Shared "enumerate and unlink the shm segments for a prefix" primitive, used by + both the cache subsystem (purge-on-disabled-start) and `traffic_ctl cache shm + clear`. Header-only since traffic_ctl does not link the cache library; tscore is + fine here, both consumers link it. + purge_segments() does no logging; it returns a report each caller formats itself. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "shared/cache_shm/Layout.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cache_shm +{ + +/// True if `pid` names a live process. EPERM counts as alive: it exists, we just may not signal it. +inline bool +process_is_alive(int32_t pid) +{ + if (pid <= 0) { + return false; + } + return ::kill(static_cast(pid), 0) == 0 || errno == EPERM; +} + +/// Outcome of trying to take the control segment's exclusive lock. +enum class LockResult { + Acquired, ///< We hold the exclusive lock; no other process does. + HeldByOther, ///< Another live process holds it (flock returned EWOULDBLOCK). + Unsupported, ///< flock is not honored for this fd (e.g. macOS POSIX shm). +}; + +/// Authoritative on Linux/tmpfs, where the lock is auto-released on crash. macOS POSIX shm returns Unsupported, so +/// callers fall back to owner_pid liveness; `unexpected_errno` distinguishes that expected case from EBADF/EINVAL/ENOLCK. +inline LockResult +try_lock_control(int fd, int *unexpected_errno = nullptr) +{ + int rc = 0; + while ((rc = ::flock(fd, LOCK_EX | LOCK_NB)) != 0 && errno == EINTR) { + ; // retry: a signal, not a real lock failure + } + if (rc == 0) { + return LockResult::Acquired; + } + // EWOULDBLOCK is the only errno meaning "another process holds it"; anything else + // means flock is unusable here -> fall back to the owner_pid backstop. + if (errno == EWOULDBLOCK) { + return LockResult::HeldByOther; + } + if (unexpected_errno != nullptr) { + *unexpected_errno = errno; + } + return LockResult::Unsupported; +} + +/// Bounded by the field size: the fixed char[] may be un-terminated in a tampered or stale segment. +inline std::string +read_shm_name(const char (&field)[MAX_SHM_NAME_LEN + 1]) +{ + return std::string(field, ::strnlen(field, sizeof(field))); +} + +/// Derived from the control-table index so the purge path can sweep the whole name space without a trustworthy table. +inline std::string +stripe_segment_name(const std::string &prefix, uint32_t stripe_index) +{ + std::string name = prefix + "s" + std::to_string(stripe_index); + if (name.size() >= MAX_SHM_NAME_LEN) { + name.resize(MAX_SHM_NAME_LEN - 1); + } + return name; +} + +/// Everything but Purged/TooSmall means nothing was unlinked. +enum class PurgeOutcome { + BadPrefix, ///< Prefix is empty or does not start with '/'. Nothing attempted. + NotPresent, ///< No control segment exists (shm_open ENOENT). Nothing to do. + OpenFailed, ///< shm_open failed for a reason other than ENOENT; cannot read safely. + MapFailed, ///< The control segment exists but could not be mmap'd. + StatFailed, ///< fstat on the control fd failed; size/validity unknown, nothing unlinked. + TooSmall, ///< Control segment is smaller than CacheShmControl; table not walked, name space swept instead. + OwnedByLive, ///< A live process owns the segment; nothing was unlinked. + Purged, ///< The stripe table was walked and its segments unlinked (possibly zero stripes). +}; + +/// One shm_unlink attempt, so callers can log each name in their own format. +struct PurgeUnlink { + std::string name; + bool is_control; ///< true for the control object, false for a stripe. + int error; ///< 0 on success; otherwise the errno from shm_unlink (ENOENT == already gone). +}; + +/// `unlinked` lists every shm_unlink attempted, stripes first, then the control object. +struct PurgeReport { + PurgeOutcome outcome = PurgeOutcome::NotPresent; + std::string control_name; ///< the control name (set whenever the prefix was valid). + int sys_errno = 0; ///< errno behind OpenFailed / MapFailed. + long long segment_size = -1; ///< control segment size in bytes; set whenever fstat succeeded. + int32_t owner_pid = 0; ///< the recorded owner pid, for OwnedByLive. + bool table_untrusted = false; ///< the stripe table could not be walked; swept `s` by name instead. + std::vector unlinked; + + /// Segments successfully removed (a shm_unlink that returned 0). + unsigned + removed() const + { + unsigned n = 0; + for (const auto &u : unlinked) { + if (u.error == 0) { + ++n; + } + } + return n; + } + + /// ENOENT is not counted: the segment being already gone is the desired end state. + unsigned + failures() const + { + unsigned n = 0; + for (const auto &u : unlinked) { + if (u.error != 0 && u.error != ENOENT) { + ++n; + } + } + return n; + } +}; + +namespace detail +{ + /// Close an fd on scope exit (the mmap survives the close). + struct FdGuard { + int fd; + ~FdGuard() + { + if (fd >= 0) { + ::close(fd); + } + } + }; +} // namespace detail + +/// Only names under `prefix` are touched, so a corrupt but magic-valid table cannot drive shm_unlink on unrelated objects. +inline void +unlink_table_stripes(const std::string &prefix, const CacheShmControl *table, std::vector &out) +{ + const uint32_t stripe_count = std::min(table->stripe_count, MAX_STRIPES); + + for (uint32_t i = 0; i < stripe_count; ++i) { + std::string name = read_shm_name(table->stripes[i].shm_name); + if (name.empty() || !name.starts_with(prefix)) { + continue; + } + int e = ::shm_unlink(name.c_str()) == 0 ? 0 : errno; + out.push_back({std::move(name), false, e}); + } +} + +/// Fallback for an unreadable stripe table: the names are ours by construction, and leaving a stripe segment behind would +/// leak the whole directory in it. Absent indices just ENOENT. +inline void +unlink_stripe_name_space(const std::string &prefix, std::vector &out) +{ + for (uint32_t i = 0; i < MAX_STRIPES; ++i) { + std::string name = stripe_segment_name(prefix, i); + if (::shm_unlink(name.c_str()) == 0) { + out.push_back({std::move(name), false, 0}); + } + } +} + +/// Unlinks the stripe segments plus the control object, unless a live process still owns it. No logging -- callers format +/// the report. The stripe table is trusted only when the magic matches and the size is ours; otherwise the name space is +/// swept instead (report.table_untrusted). See "Operator tooling" in the shm-fast-restart developer guide. +inline PurgeReport +purge_segments(const std::string &prefix) +{ + PurgeReport report; + + if (prefix.empty() || prefix[0] != '/') { + report.outcome = PurgeOutcome::BadPrefix; + return report; + } + report.control_name = control_segment_name(prefix); + + // Records the name-sweep fallback where the fact is established, so the flag cannot + // disagree with the outcome that led to it. + auto sweep_stripe_name_space = [&report, &prefix]() { + report.table_untrusted = true; + unlink_stripe_name_space(prefix, report.unlinked); + }; + + int fd = ::shm_open(report.control_name.c_str(), O_RDONLY, 0); + if (fd < 0) { + report.sys_errno = errno; + report.outcome = (errno == ENOENT) ? PurgeOutcome::NotPresent : PurgeOutcome::OpenFailed; + return report; + } + detail::FdGuard guard{fd}; + + // clang-format off + struct stat sb{}; + // clang-format on + if (::fstat(fd, &sb) < 0) { + // Size and validity are unknown: report the error and leave the segment + // alone rather than unlink a control object we failed to stat. + report.sys_errno = errno; + report.outcome = PurgeOutcome::StatFailed; + return report; + } + report.segment_size = static_cast(sb.st_size); + if (static_cast(sb.st_size) < CONTROL_SIZE) { + // Too small to hold this build's header/table: there is no table to walk, so + // sweep the stripe name space and unlink the control object itself. + report.outcome = PurgeOutcome::TooSmall; + sweep_stripe_name_space(); + int e = ::shm_unlink(report.control_name.c_str()) == 0 ? 0 : errno; + report.unlinked.push_back({report.control_name, true, e}); + return report; + } + + // Larger than this build's page-rounded CONTROL_SIZE means a build with a different + // sizeof(CacheShmControl) wrote it. The frozen header prefix is still readable (so + // the owner guard below applies), but stripes[] may have a different stride entirely, + // so its names must not drive shm_unlink. + const bool own_size = is_own_control_size(static_cast(sb.st_size)); + + void *addr = ::mmap(nullptr, CONTROL_SIZE, PROT_READ, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + report.sys_errno = errno; + report.outcome = PurgeOutcome::MapFailed; + return report; + } + + const auto *ctrl = static_cast(addr); + const bool magic_ok = std::memcmp(ctrl->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)) == 0; + + const LockResult lock = try_lock_control(fd); + if (lock == LockResult::HeldByOther || (lock == LockResult::Unsupported && magic_ok && process_is_alive(ctrl->owner_pid))) { + report.owner_pid = magic_ok ? ctrl->owner_pid : 0; + report.outcome = PurgeOutcome::OwnedByLive; + ::munmap(addr, CONTROL_SIZE); + return report; + } + + if (magic_ok && own_size) { + unlink_table_stripes(prefix, ctrl, report.unlinked); + } else { + // Bad magic, or a foreign sizeof(CacheShmControl): the table cannot be walked. + sweep_stripe_name_space(); + } + ::munmap(addr, CONTROL_SIZE); + + int e = ::shm_unlink(report.control_name.c_str()) == 0 ? 0 : errno; + report.unlinked.push_back({report.control_name, true, e}); + + report.outcome = PurgeOutcome::Purged; + return report; +} + +} // namespace cache_shm diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index c914eae89d3..fe19d38e484 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -151,6 +151,7 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine01 TS_HAS_SO_PEERCRED #cmakedefine01 TS_HAS_TESTS #cmakedefine01 TS_USE_DIAGS +#cmakedefine01 TS_USE_CACHE_SHM #cmakedefine01 TS_USE_EPOLL #cmakedefine01 TS_USE_FAST_SDK #cmakedefine01 TS_ENABLE_FIPS diff --git a/src/iocore/cache/AggregateWriteBuffer.cc b/src/iocore/cache/AggregateWriteBuffer.cc index b761d656170..1407d5b2693 100644 --- a/src/iocore/cache/AggregateWriteBuffer.cc +++ b/src/iocore/cache/AggregateWriteBuffer.cc @@ -49,7 +49,6 @@ AggregateWriteBuffer::flush(int fd, off_t write_pos) const { int r = pwrite(fd, this->_buffer, this->_buffer_pos, write_pos); if (r != this->_buffer_pos) { - ink_assert(!"flushing agg buffer failed"); return false; } return true; diff --git a/src/iocore/cache/AggregateWriteBuffer.h b/src/iocore/cache/AggregateWriteBuffer.h index ad99b03ce04..22951fb797a 100644 --- a/src/iocore/cache/AggregateWriteBuffer.h +++ b/src/iocore/cache/AggregateWriteBuffer.h @@ -120,7 +120,7 @@ class AggregateWriteBuffer * @param write_pos The offset at which to write the buffer data. * @return Returns true if all bytes were flushed, otherwise false. */ - bool flush(int fd, off_t write_pos) const; + [[nodiscard]] bool flush(int fd, off_t write_pos) const; /** * Copy part of the buffer. diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index f8a252b430c..28695107d37 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -27,6 +27,7 @@ add_library( CacheHttp.cc CacheProcessor.cc CacheRead.cc + CacheShm.cc CacheVC.cc CacheWrite.cc HttpTransactCache.cc @@ -92,6 +93,11 @@ if(BUILD_TESTING) add_cache_test(Update_Header unit_tests/test_Update_header.cc) add_cache_test(CacheStripe unit_tests/test_Stripe.cc) add_cache_test(CacheAggregateWriteBuffer unit_tests/test_AggregateWriteBuffer.cc) + # Only the shutdown test attaches a live segment; the rest need no shm syscall. + add_cache_test(CacheShm unit_tests/test_CacheShm.cc) + if(TS_USE_CACHE_SHM) + add_cache_test(CacheShmShutdown unit_tests/test_CacheShmShutdown.cc) + endif() # Unit Tests without unit_tests/main.cc add_executable(test_ConfigVolumes unit_tests/test_ConfigVolumes.cc) diff --git a/src/iocore/cache/CacheDir.cc b/src/iocore/cache/CacheDir.cc index 7368e57c4e1..a5746297a20 100644 --- a/src/iocore/cache/CacheDir.cc +++ b/src/iocore/cache/CacheDir.cc @@ -27,6 +27,7 @@ #include "P_CacheInternal.h" #include "PreservationTable.h" #include "Stripe.h" +#include "CacheShm.h" #include "tscore/hugepages.h" #include "tscore/Random.h" @@ -274,24 +275,33 @@ Directory::bucket_length(Dir *b, int s) return i; } +int +Directory::check_segment(int s) +{ + Dir *seg = this->get_segment(s); + + for (int i = 0; i < this->buckets; i++) { + Dir *b = dir_bucket(i, seg); + if (!(this->bucket_length(b, s) >= 0)) { + return 0; + } + if (!(!dir_next(b) || dir_offset(b))) { + return 0; + } + if (!(dir_bucket_loop_check(b, seg))) { + return 0; + } + } + return 1; +} + int Directory::check() { - int i, s; Dbg(dbg_ctl_cache_check_dir, "inside check dir"); - for (s = 0; s < this->segments; s++) { - Dir *seg = this->get_segment(s); - for (i = 0; i < this->buckets; i++) { - Dir *b = dir_bucket(i, seg); - if (!(this->bucket_length(b, s) >= 0)) { - return 0; - } - if (!(!dir_next(b) || dir_offset(b))) { - return 0; - } - if (!(dir_bucket_loop_check(b, seg))) { - return 0; - } + for (int s = 0; s < this->segments; s++) { + if (!this->check_segment(s)) { + return 0; } } return 1; @@ -948,6 +958,11 @@ sync_cache_dir_on_shutdown() thr.join(); } + // Each stripe was snapshotted above. This does not stop future writers (the event system + // is still up), but a late or in-flight write only leaves a dir entry the next start's + // magic+key read check treats as a miss -- never served corruption -- so mark clean here. + CacheShm::mark_clean_shutdown(); + Dbg(dbg_ctl_cache_dir_sync, "shutdown sync done"); } diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index 1fa158d9889..c178aeb959d 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -28,6 +28,7 @@ #include "P_CacheInternal.h" #include "StripeSM.h" #include "Stripe.h" +#include "CacheShm.h" // Must be included after P_CacheInternal.h. #include "P_CacheHosting.h" @@ -187,6 +188,9 @@ CacheProcessor::start_internal(int flags) gndisks = theCacheStore.n_spans; gdisks.resize(gndisks); + // Must run before any Stripe is constructed so each can attach/create its segment. + CacheShm::initialize(theCacheStore); + // Temporaries to carry values between loops char **paths = static_cast(alloca(sizeof(char *) * gndisks)); memset(paths, 0, sizeof(char *) * gndisks); @@ -1495,6 +1499,9 @@ CacheProcessor::cacheInitialized() } } + // All stripes have claimed their segments; reclaim any orphan (e.g. a dropped disk). + CacheShm::finalize_attach(); + if (caches_ready) { Dbg(dbg_ctl_cache_init, "CacheProcessor::cacheInitialized - caches_ready=0x%0X, gnvol=%d", (unsigned int)caches_ready, gnstripes.load()); diff --git a/src/iocore/cache/CacheShm.cc b/src/iocore/cache/CacheShm.cc new file mode 100644 index 00000000000..a586a58feb1 --- /dev/null +++ b/src/iocore/cache/CacheShm.cc @@ -0,0 +1,932 @@ +/** @file + + Shared-memory-backed cache directory for fast restart. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "CacheShm.h" +#include "shared/cache_shm/Layout.h" +#include "shared/cache_shm/Purge.h" + +#include "P_CacheDir.h" +#include "iocore/cache/Store.h" + +#include "records/RecCore.h" +#include "tscore/Diags.h" +#include "tscore/HashFNV.h" +#include "tscore/hugepages.h" +#include "tscore/ink_align.h" +#include "tscore/ink_config.h" +#include "tscore/ink_memory.h" +#include "tscore/ink_string.h" +#include "tsutil/DbgCtl.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Always compiled: none of these touches an shm syscall, so a build without POSIX shm still gets the real fingerprints +// and their test coverage rather than a stub reporting a zero ABI hash. + +CacheShm::Mode CacheShm::_mode = CacheShm::Mode::Disabled; + +namespace +{ +void +fnv_update(ATSHash64FNV1a &h, uint64_t v) +{ + h.update(&v, sizeof v); +} +} // namespace + +uint64_t +CacheShm::abi_hash() +{ + ATSHash64FNV1a h; + h.update(tag.data(), tag.size()); + fnv_update(h, sizeof(Dir)); + fnv_update(h, sizeof(StripeHeaderFooter)); + fnv_update(h, sizeof(cache_shm::CacheShmControl)); + fnv_update(h, sizeof(cache_shm::StripeEntry)); + fnv_update(h, DIR_DEPTH); + fnv_update(h, SIZEOF_DIR); + fnv_update(h, cache_shm::MAX_STRIPES); + return h.get(); +} + +uint64_t +CacheShm::storage_signature(const Store &store) +{ + ATSHash64FNV1a h; + for (unsigned i = 0; i < store.n_spans; ++i) { + const Span *span = store.spans[i]; + if (span == nullptr) { + continue; + } + if (span->pathname) { + std::string_view path{span->pathname.get()}; + h.update(path.data(), path.size()); + } + fnv_update(h, static_cast(span->blocks)); + fnv_update(h, static_cast(span->offset)); + fnv_update(h, static_cast(span->hw_sector_size)); + } + return h.get(); +} + +bool +CacheShm::process_is_alive(int pid) +{ + return cache_shm::process_is_alive(pid); +} + +#if !TS_USE_CACHE_SHM + +// No POSIX shm in libc; glibc < 2.34 puts shm_open/shm_unlink in librt, which ATS does not link. Mode stays Disabled, so +// every stripe takes the heap path and the cache behaves as it did before this feature existed. + +void +CacheShm::initialize(const Store &) +{ + // Only when the operator asked for it; the default is off, so an unconditional note would be noise on every start. + if (RecGetRecordInt("proxy.config.cache.shm.enabled").value_or(0) != 0) { + Warning("cache shm: proxy.config.cache.shm.enabled is set, but this build has no POSIX shared memory support " + "(shm_open is not in libc); using heap directories"); + } +} + +char * +CacheShm::attach_or_create_stripe(const char *, std::size_t) +{ + return nullptr; +} + +void +CacheShm::finalize_attach() +{ +} + +bool +CacheShm::is_shm_pointer(char *) +{ + return false; +} + +void +CacheShm::mark_clean_shutdown() +{ +} + +void +CacheShm::invalidate_stripe_directory(char *) +{ +} + +void +CacheShm::detach_stripe(char *) +{ +} + +#else + +namespace +{ + +DbgCtl dbg_ctl{"cache_shm"}; + +using cache_shm::CACHE_SHM_MAGIC; +using cache_shm::CACHE_SHM_SCHEMA_VERSION; +using cache_shm::CacheShmControl; +using cache_shm::CONTROL_HEADER_SIZE; +using cache_shm::control_segment_name; +using cache_shm::CONTROL_SIZE; +using cache_shm::LockResult; +using cache_shm::MAX_SHM_NAME_LEN; +using cache_shm::MAX_STRIPES; +using cache_shm::read_shm_name; +using cache_shm::StripeEntry; +using cache_shm::try_lock_control; + +// Sanity bound: the control struct (header + stripe table) must stay small. +constexpr std::size_t MAX_CONTROL_SEGMENT_BYTES = 32 * 1024; +static_assert(sizeof(CacheShmControl) <= MAX_CONTROL_SEGMENT_BYTES, "control segment unexpectedly large"); + +// Configuration loaded at initialize() time. +struct Config { + bool enabled = false; + bool use_hugepages = false; + bool purge_stale_on_start = false; + std::string name_prefix; // normalized "/-" (see normalize_name_prefix); set in load_config. +}; + +Config g_config; + +// Live state for the open control segment. +CacheShmControl *g_control = nullptr; +std::string g_control_name; + +// Held for the process lifetime so the OS releases it on exit. Only set on the path that owns the segment. +ats_scoped_fd g_control_fd; + +// Pointers we returned, so ~Stripe can choose munmap over ats_free and unmap the right span, and so an invalidation can +// reach the stripe's control-table entry without every caller having to carry the index. +struct MappedStripe { + std::size_t size; + uint32_t index; +}; +std::mutex g_pointers_mutex; +std::unordered_map g_pointers; + +// Guards the stripe table and the claim bookkeeping below; stripes initialize concurrently across disk threads. +std::mutex g_table_mutex; + +// Per-run partial-attach bookkeeping, indexed in lockstep with g_control->stripes[]. +// An entry still unclaimed once init completes is an orphan for finalize_attach(). Process-local, reset each run. +bool g_entry_claimed[MAX_STRIPES] = {}; +uint32_t g_claims_this_run = 0; + +/// Full 64-bit stripe identity used to match a stripe to its prior shm segment. +uint64_t +compute_stripe_key_hash(const char *stripe_key) +{ + ATSHash64FNV1a hash; + hash.update(stripe_key, std::strlen(stripe_key)); + return hash.get(); +} + +/// Shared with the purge path, which sweeps the index space by name when the stripe table cannot be trusted. +using cache_shm::stripe_segment_name; + +// Named so the two cannot be transposed at a call site. +enum class ShmAccess { Open, Create }; +enum class HugePages { Off, On }; + +/// nullptr on failure. `out_fd`, when set, keeps the fd open so the caller can flock it; otherwise it is closed, which +/// the mapping survives. +void * +open_and_map_shm(const std::string &name, std::size_t size, ShmAccess access, [[maybe_unused]] HugePages hugepages, + int *out_fd = nullptr, int *out_errno = nullptr) +{ + if (out_errno != nullptr) { + *out_errno = 0; + } + int oflags = O_RDWR; + if (access == ShmAccess::Create) { + // O_EXCL so a create never adopts a pre-existing (attacker-planted) object. + oflags |= O_CREAT | O_EXCL; + } + + ats_scoped_fd fd{shm_open(name.c_str(), oflags, 0600)}; + if (fd < 0) { + int e = errno; + Dbg(dbg_ctl, "shm_open(%s, %s) failed: %s", name.c_str(), access == ShmAccess::Create ? "create" : "open", strerror(e)); + if (out_errno != nullptr) { + *out_errno = e; + } + return nullptr; + } + + if (access == ShmAccess::Create) { + if (ftruncate(fd, size) < 0) { + int e = errno; + Warning("ftruncate(%s, %zu) failed: %s", name.c_str(), size, strerror(e)); + shm_unlink(name.c_str()); + if (out_errno != nullptr) { + *out_errno = e; + } + return nullptr; + } + } else { + // The kernel rounds an shm object up to a page, so accept any size in [requested, page-up]. + struct stat sb { + }; + std::size_t expected_max = INK_ALIGN(size, ats_pagesize()); + if (fstat(fd, &sb) < 0 || sb.st_size < 0 || static_cast(sb.st_size) < size || + static_cast(sb.st_size) > expected_max) { + Dbg(dbg_ctl, "shm %s size mismatch (have %lld, want %zu, max %zu)", name.c_str(), static_cast(sb.st_size), size, + expected_max); + return nullptr; + } + } + + int prot = PROT_READ | PROT_WRITE; + int flags = MAP_SHARED; + void *addr = mmap(nullptr, size, prot, flags, fd, 0); + if (addr == MAP_FAILED) { + int e = errno; + Warning("mmap(%s, %zu) failed: %s", name.c_str(), size, strerror(e)); + // Or the leak wedges the next O_EXCL create on EEXIST. + if (access == ShmAccess::Create) { + shm_unlink(name.c_str()); + } + if (out_errno != nullptr) { + *out_errno = e; + } + return nullptr; + } + + // MAP_HUGETLB is not usable on a tmpfs-backed fd, so advise THP instead; needs shmem THP enabled on the host. +#if defined(MADV_HUGEPAGE) + if (hugepages == HugePages::On) { + if (madvise(addr, size, MADV_HUGEPAGE) != 0) { + Dbg(dbg_ctl, "madvise(MADV_HUGEPAGE) on %s failed: %s", name.c_str(), strerror(errno)); + } + } +#endif + + if (out_fd != nullptr) { + *out_fd = fd.release(); // caller owns the fd and keeps it open for flock + } + return addr; +} + +/// How the pre-existing control segment could be mapped. +enum class ControlMap { + Absent, ///< No segment with this name (ENOENT); nothing to attach. + Full, ///< This build's size, fully mapped: eligible for the trust gates. + Foreign, ///< Another build's size: only the frozen header is mapped (if it even + ///< fits). Never trusted -- guard the owner, then drop and recreate. + Failed, ///< Exists but is unusable (permissions, fstat or mmap failure). +}; + +struct ControlOpen { + ControlMap map = ControlMap::Absent; + CacheShmControl *ctrl = nullptr; ///< nullptr when Absent/Failed, or Foreign and too small for the header. + std::size_t mapped = 0; ///< length to munmap. + std::size_t size = 0; ///< the segment's actual size, for the Foreign diagnostic. + int fd = -1; ///< open on Full/Foreign so the caller can flock it. + int sys_err = 0; ///< errno behind Absent/Failed. +}; + +/// Not routed through open_and_map_shm: failing on a foreign size would leave the O_EXCL create below wedged on EEXIST +/// every restart until an operator ran `traffic_ctl cache shm clear`. Mapping just the frozen header instead keeps the +/// owner guard usable so the segment can be dropped. +ControlOpen +open_control_segment(const std::string &name) +{ + ControlOpen out; + ats_scoped_fd fd{shm_open(name.c_str(), O_RDWR, 0600)}; + if (fd < 0) { + out.sys_err = errno; + out.map = out.sys_err == ENOENT ? ControlMap::Absent : ControlMap::Failed; + Dbg(dbg_ctl, "shm_open(%s, open) failed: %s", name.c_str(), strerror(out.sys_err)); + return out; + } + + // clang-format off + struct stat sb{}; + // clang-format on + if (fstat(fd, &sb) < 0) { + out.sys_err = errno; + out.map = ControlMap::Failed; + Warning("cache shm: fstat(%s) failed: %s", name.c_str(), strerror(out.sys_err)); + return out; + } + + const std::size_t actual = sb.st_size < 0 ? 0 : static_cast(sb.st_size); + const bool own_size = cache_shm::is_own_control_size(actual); + + out.map = own_size ? ControlMap::Full : ControlMap::Foreign; + out.mapped = own_size ? CONTROL_SIZE : std::min(actual, CONTROL_HEADER_SIZE); + out.size = actual; + + if (out.mapped >= CONTROL_HEADER_SIZE) { + void *addr = mmap(nullptr, out.mapped, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + out.sys_err = errno; + out.map = ControlMap::Failed; + Warning("mmap(%s, %zu) failed: %s", name.c_str(), out.mapped, strerror(out.sys_err)); + out.mapped = 0; + return out; + } + out.ctrl = static_cast(addr); + } else { + // No ATS build wrote this, so there is nothing to read and the flock is the only owner guard left. + out.mapped = 0; + } + + out.fd = fd.release(); // caller owns it: flock, then close or keep for the run + return out; +} + +// `table` is nullptr when the stripe table may not be read (foreign layout or bad magic), which sweeps the name space +// instead so a stripe segment -- and the whole directory in it -- is never left behind. +void +unlink_all_known_segments(void *mapping, std::size_t mapping_len, const CacheShmControl *table) +{ + // Shares purge_segments()'s primitives so the prefix filter guarding them cannot drift. Cannot call purge_segments() + // itself: this path already holds the control fd, mapping, and the exclusive lock that makes unlinking safe. + std::vector unlinked; + + if (table != nullptr) { + cache_shm::unlink_table_stripes(g_config.name_prefix, table, unlinked); + } else { + cache_shm::unlink_stripe_name_space(g_config.name_prefix, unlinked); + } + for (const auto &u : unlinked) { + Dbg(dbg_ctl, "shm_unlink stripe %s%s", u.name.c_str(), table != nullptr ? "" : " (table untrusted; swept by name)"); + } + + if (mapping != nullptr) { + munmap(mapping, mapping_len); + } + g_control = nullptr; + if (!g_control_name.empty()) { + Dbg(dbg_ctl, "shm_unlink control %s", g_control_name.c_str()); + shm_unlink(g_control_name.c_str()); + } +} + +// Opt-in and best-effort: logs but never blocks startup. Shares the enumerate-and-unlink work with `traffic_ctl cache +// shm clear`; this only renders the result into diags. +void +purge_stale_segments(const std::string &prefix) +{ + const cache_shm::PurgeReport report = cache_shm::purge_segments(prefix); + + switch (report.outcome) { + case cache_shm::PurgeOutcome::BadPrefix: + // load_config() already warned about a bad prefix; stay quiet here. + case cache_shm::PurgeOutcome::NotPresent: + return; // ENOENT: shm never used with this prefix. + case cache_shm::PurgeOutcome::OpenFailed: + Warning("cache shm: cannot open control segment %s to purge stale segments: %s", report.control_name.c_str(), + strerror(report.sys_errno)); + return; + case cache_shm::PurgeOutcome::MapFailed: + Warning("cache shm: mmap of control segment %s failed while purging: %s", report.control_name.c_str(), + strerror(report.sys_errno)); + return; + case cache_shm::PurgeOutcome::StatFailed: + Warning("cache shm: cannot stat control segment %s to purge stale segments: %s", report.control_name.c_str(), + strerror(report.sys_errno)); + return; + case cache_shm::PurgeOutcome::TooSmall: + Warning("cache shm: leftover control segment %s is too small to read (%lld bytes); unlinking it", report.control_name.c_str(), + report.segment_size); + break; // purge_segments() already unlinked the control object; render the result below. + case cache_shm::PurgeOutcome::OwnedByLive: + Warning("cache shm: control segment %s is owned by a live process; leaving stale segments in place", + report.control_name.c_str()); + return; + case cache_shm::PurgeOutcome::Purged: + break; + } + + if (report.table_untrusted) { + Warning("cache shm: leftover control segment %s has an unreadable stripe table (%lld bytes); swept the '%ss' name space", + report.control_name.c_str(), report.segment_size, prefix.c_str()); + } + for (const auto &u : report.unlinked) { + if (u.error == 0) { + Dbg(dbg_ctl, "purge: unlinked %s %s", u.is_control ? "control" : "stripe", u.name.c_str()); + } else if (u.error != ENOENT) { + Warning("cache shm: failed to unlink %s %s while purging: %s", u.is_control ? "control segment" : "stripe", u.name.c_str(), + strerror(u.error)); + } + } + + Note("cache shm: purged stale segments while disabled (removed %u, %u failure(s), prefix '%s')", report.removed(), + report.failures(), prefix.c_str()); +} + +bool +load_config() +{ + RecInt enabled = RecGetRecordInt("proxy.config.cache.shm.enabled").value_or(0); + g_config.enabled = enabled != 0; + + RecInt use_hugepages = RecGetRecordInt("proxy.config.cache.shm.use_hugepages").value_or(0); + g_config.use_hugepages = use_hugepages != 0; + + // Inherit the global hugepage intent unless the operator set the shm knob explicitly. + RecSourceT hp_source = REC_SOURCE_NULL; + if (!g_config.use_hugepages && ats_hugepage_enabled() && + RecGetRecordSource("proxy.config.cache.shm.use_hugepages", &hp_source) == REC_ERR_OKAY && hp_source == REC_SOURCE_DEFAULT) { + g_config.use_hugepages = true; + } + + RecInt purge_stale_on_start = RecGetRecordInt("proxy.config.cache.shm.purge_stale_on_start").value_or(0); + g_config.purge_stale_on_start = purge_stale_on_start != 0; + + char prefix_buf[256] = {0}; + std::string configured = "ats"; // operator sets only the middle word; framing is added below. + if (RecGetRecordString("proxy.config.cache.shm.name_prefix", prefix_buf, sizeof(prefix_buf)).has_value() && + prefix_buf[0] != '\0') { + configured = prefix_buf; + } + g_config.name_prefix = cache_shm::normalize_name_prefix(configured); + + return g_config.enabled; +} + +// Marks the slot non-empty so a concurrent create cannot pick the same index. MAX_STRIPES when full. Caller must hold +// g_table_mutex. +uint32_t +reserve_stripe_slot(uint64_t key_hash, std::size_t directory_size, std::string &out_name) +{ + uint32_t idx = g_control->stripe_count; + bool reuse_slot = false; + for (uint32_t i = 0; i < g_control->stripe_count && i < MAX_STRIPES; ++i) { + if (g_control->stripes[i].shm_name[0] == '\0') { + idx = i; + reuse_slot = true; + break; + } + } + if (!reuse_slot && g_control->stripe_count >= MAX_STRIPES) { + Warning("cache shm: stripe count exceeds MAX_STRIPES (%zu); falling back", MAX_STRIPES); + return MAX_STRIPES; + } + + out_name = stripe_segment_name(g_config.name_prefix, idx); + if (!reuse_slot) { + g_control->stripe_count++; + } + StripeEntry &e = g_control->stripes[idx]; + ink_strlcpy(e.shm_name, out_name.c_str(), sizeof(e.shm_name)); + e.raw_dir_size = directory_size; + e.stripe_key_hash = key_hash; + e.dir_untrusted = 0; // fresh segment, so any mark from a prior occupant of this slot is stale + return idx; +} + +// Tombstones the slot for reuse. Caller must hold g_table_mutex. +void +release_reserved_slot(uint32_t idx) +{ + StripeEntry &e = g_control->stripes[idx]; + e.shm_name[0] = '\0'; + e.raw_dir_size = 0; + e.stripe_key_hash = 0; + e.dir_untrusted = 0; +} + +// Takes the locks itself, so the shm syscalls that produced `p` could run with g_table_mutex dropped. +char * +claim_mapped_stripe(uint32_t idx, void *p, std::size_t size) +{ + { + std::scoped_lock lk{g_table_mutex}; + g_entry_claimed[idx] = true; + ++g_claims_this_run; + } + { + std::scoped_lock plk{g_pointers_mutex}; + g_pointers.insert({ + static_cast(p), MappedStripe{size, idx} + }); + } + return static_cast(p); +} + +} // namespace + +void +CacheShm::initialize(const Store &store) +{ + if (!load_config()) { + _mode = Mode::Disabled; + // Leftovers would keep consuming memory, and a later re-enable would attach a directory that went stale meanwhile. + if (g_config.purge_stale_on_start) { + purge_stale_segments(g_config.name_prefix); + } + Dbg(dbg_ctl, "shm disabled"); + return; + } + + // Surface the MAP_HUGETLB -> THP substitution once so it isn't a silent downgrade. + if (ats_hugepage_enabled()) { + if (g_config.use_hugepages) { + Note("cache shm: global hugepages enabled; MAP_HUGETLB is not usable for the tmpfs-backed dir, " + "advising MADV_HUGEPAGE (transparent huge pages) on the mapping instead"); + } else { + Warning("cache shm: global hugepages enabled but proxy.config.cache.shm.use_hugepages is 0; " + "the tmpfs-backed dir will use base pages (MAP_HUGETLB does not apply to shm)"); + } + } + + g_control_name = control_segment_name(g_config.name_prefix); + if (g_control_name.size() >= MAX_SHM_NAME_LEN) { + Warning("shm name_prefix too long (control segment name '%s' exceeds %zu chars); shm disabled", g_control_name.c_str(), + MAX_SHM_NAME_LEN); + _mode = Mode::Disabled; + return; + } + + const uint64_t expected_abi = abi_hash(); + const uint64_t expected_signature = storage_signature(store); + + // Try to attach an existing control segment first. + ControlOpen opened = open_control_segment(g_control_name); + if (opened.map == ControlMap::Failed) { + Warning("cache shm: cannot use existing control segment %s: %s; shm disabled", g_control_name.c_str(), + strerror(opened.sys_err)); + _mode = Mode::Disabled; + return; + } + if (opened.map != ControlMap::Absent) { + CacheShmControl *ctrl = opened.ctrl; + + // Refuse shm, and rebuild from disk, if another live process still owns this segment. + int flock_errno = 0; + const LockResult lock = try_lock_control(opened.fd, &flock_errno); + bool live_owner = false; + switch (lock) { + case LockResult::Acquired: + break; // we hold the exclusive lock, so any prior owner is gone + case LockResult::HeldByOther: + live_owner = true; + break; + case LockResult::Unsupported: // macOS POSIX shm: flock is a no-op, fall back to owner_pid + Dbg(dbg_ctl, "flock unsupported for control segment %s (errno %d: %s); using owner-pid liveness guard", + g_control_name.c_str(), flock_errno, strerror(flock_errno)); + live_owner = ctrl != nullptr && ctrl->owner_pid != 0 && ctrl->owner_pid != static_cast(getpid()) && + process_is_alive(ctrl->owner_pid); + break; + } + if (live_owner) { + Warning("cache shm: control segment %s has a live owner (pid %d); disabling shm this run to avoid concurrent attach", + g_control_name.c_str(), ctrl != nullptr ? ctrl->owner_pid : 0); + if (ctrl != nullptr) { + munmap(ctrl, opened.mapped); + } + close(opened.fd); + _mode = Mode::Disabled; + return; + } + + // Only the frozen header was mapped; the stripe table behind it may have a different layout entirely. + if (opened.map == ControlMap::Foreign) { + Note("cache shm: control segment %s is %zu bytes, not this build's %zu; dropping it", g_control_name.c_str(), opened.size, + CONTROL_SIZE); + } + const bool magic_ok = opened.map == ControlMap::Full && std::memcmp(ctrl->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)) == 0; + + bool ok = magic_ok; + if (ok && ctrl->schema_version != CACHE_SHM_SCHEMA_VERSION) { + Note("cache shm: schema mismatch (%u vs %u), dropping", ctrl->schema_version, CACHE_SHM_SCHEMA_VERSION); + ok = false; + } + if (ok && ctrl->abi_hash != expected_abi) { + Note("cache shm: ABI mismatch, dropping"); + ok = false; + } + + // Not a hard gate: a storage change keeps the segment and each stripe attaches by its own identity. + const bool storage_changed = ok && ctrl->storage_signature != expected_signature; + + if (ok && ctrl->clean_shutdown == 0) { + // A crash may have left dir entries pointing at content never flushed, so no + // stripe can safely skip recovery -- whole-segment drop. + Note("cache shm: previous run did not shutdown cleanly, dropping"); + ok = false; + } + + if (ok) { + Note("cache shm: attaching up to %u stripes (fast restart%s)", ctrl->stripe_count, + storage_changed ? ", partial -- storage changed" : ""); + g_control = ctrl; + g_control_fd = opened.fd; // hold the exclusive lock for the process lifetime + std::memset(g_entry_claimed, 0, sizeof(g_entry_claimed)); + g_claims_this_run = 0; + if (storage_changed) { + g_control->storage_signature = expected_signature; + } + // Become owner and clear clean_shutdown so a crash this run drops shm next time. + g_control->owner_pid = static_cast(getpid()); + g_control->clean_shutdown = 0; + msync(g_control, CONTROL_SIZE, MS_SYNC); + _mode = Mode::AttachExisting; + return; + } + + // Drop everything and fall through to fresh-create. We hold the exclusive lock, + // so unlinking cannot pull segments out from under a live owner. + unlink_all_known_segments(ctrl, opened.mapped, magic_ok ? ctrl : nullptr); + close(opened.fd); // releases the lock on the now-unlinked object + } + + // Create fresh control segment. + int fresh_fd = -1; + int create_errno = 0; + void *fresh = open_and_map_shm(g_control_name, CONTROL_SIZE, ShmAccess::Create, HugePages::Off, &fresh_fd, &create_errno); + if (fresh == nullptr) { + // Surface the errno + offending name: e.g. an embedded '/' in name_prefix yields EINVAL here. + Warning("cache shm: failed to create control segment %s: %s; shm disabled", g_control_name.c_str(), strerror(create_errno)); + _mode = Mode::Disabled; + return; + } + // Lock the freshly created segment. Another starting process could have created + // and locked it first in the window since the drop above; if so, refuse. + if (try_lock_control(fresh_fd) == LockResult::HeldByOther) { + Warning("cache shm: lost the create race for control segment %s; disabling shm this run", g_control_name.c_str()); + munmap(fresh, CONTROL_SIZE); + close(fresh_fd); + _mode = Mode::Disabled; + return; + } + g_control = static_cast(fresh); + g_control_fd = fresh_fd; // hold the exclusive lock for the process lifetime + std::memset(g_control, 0, CONTROL_SIZE); + std::memset(g_entry_claimed, 0, sizeof(g_entry_claimed)); + g_claims_this_run = 0; + std::memcpy(g_control->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)); + g_control->schema_version = CACHE_SHM_SCHEMA_VERSION; + g_control->abi_hash = expected_abi; + g_control->storage_signature = expected_signature; + g_control->clean_shutdown = 0; + g_control->owner_pid = static_cast(getpid()); + g_control->stripe_count = 0; + + _mode = Mode::CreateFresh; + Note("cache shm: creating fresh control segment %s (owner pid %d)", g_control_name.c_str(), static_cast(getpid())); + return; +} + +char * +CacheShm::attach_or_create_stripe(const char *stripe_key, std::size_t directory_size) +{ + if (_mode == Mode::Disabled || g_control == nullptr) { + return nullptr; + } + + const uint64_t key_hash = compute_stripe_key_hash(stripe_key); + const HugePages hugepages = g_config.use_hugepages ? HugePages::On : HugePages::Off; + + // Decide under the table lock, then run the shm syscalls with it dropped (holding + // it would serialize every disk thread's init; each stripe owns a distinct segment). + std::string attach_name; // non-empty => map this existing segment + std::string create_name; // set when a fresh slot was reserved (the create path) + uint32_t idx = MAX_STRIPES; + { + std::scoped_lock lk{g_table_mutex}; + + // 1. Try to attach this stripe's prior segment, matched by 64-bit identity (not + // name), so a span going offline shifts indices but not identities. + for (uint32_t i = 0; i < g_control->stripe_count && i < MAX_STRIPES; ++i) { + StripeEntry &e = g_control->stripes[i]; + if (e.shm_name[0] == '\0' || e.stripe_key_hash != key_hash) { + continue; // tombstoned slot, or a different stripe + } + if (g_entry_claimed[i]) { + // Another stripe this run already took this entry, so two distinct stripes + // hashed to one identity (duplicate hash_text, or a 64-bit FNV-1a collision). + // Sharing one directory between them would corrupt both; create fresh instead. + Warning("cache shm: stripe key collision on %s; creating a fresh segment for key=%s", read_shm_name(e.shm_name).c_str(), + stripe_key); + break; + } + if (e.dir_untrusted) { + // Last shutdown could not vouch for this directory (bad disk, or a write still in flight). Tombstone it here rather + // than leaving an orphan for finalize_attach: the slot is reused immediately, so stripe_count cannot creep across + // runs that never reach finalize, and the old segment never coexists with its replacement. + // The unlink stays under the lock even though the other shm syscalls do not: once the slot is a tombstone another + // disk thread can reserve it and derive this same name, and unlinking after that would strip the name off the + // segment it just created. + const std::string name = read_shm_name(e.shm_name); + Note("cache shm: stripe %s was marked untrusted at shutdown; recreating", name.c_str()); + shm_unlink(name.c_str()); + release_reserved_slot(i); + break; + } + if (e.raw_dir_size != directory_size) { + // Same identity, different size: shouldn't happen (size derives from the + // keyed blocks). Treat as a miss and recreate; the stale entry is reaped by + // finalize_attach(). + Note("cache shm: stripe %s size mismatch (have %llu, want %zu); recreating", read_shm_name(e.shm_name).c_str(), + static_cast(e.raw_dir_size), directory_size); + break; + } + attach_name = read_shm_name(e.shm_name); + idx = i; + break; + } + + // 2. No usable prior segment -- reserve a slot for a fresh create under the lock. + if (attach_name.empty() && (idx = reserve_stripe_slot(key_hash, directory_size, create_name)) == MAX_STRIPES) { + return nullptr; // table full (already logged) + } + } + + // Attach path: map the existing segment outside the lock. + if (!attach_name.empty()) { + void *p = open_and_map_shm(attach_name, directory_size, ShmAccess::Open, hugepages); + if (p != nullptr) { + Note("cache shm: attached stripe %s (%zu bytes) for key=%s", attach_name.c_str(), directory_size, stripe_key); + return claim_mapped_stripe(idx, p, directory_size); + } + // Attach failed (segment vanished/unmappable): reserve a fresh slot and fall + // through to create. The stale entry is reaped by finalize_attach(). + Note("cache shm: failed to attach stripe %s; recreating", attach_name.c_str()); + std::scoped_lock lk{g_table_mutex}; + if ((idx = reserve_stripe_slot(key_hash, directory_size, create_name)) == MAX_STRIPES) { + return nullptr; + } + } + + // Create path: slot already reserved; syscalls run outside the lock. A fresh + // ftruncate'd segment is zero-filled (magic 0), so Stripe::init falls back to the + // disk read and repopulates it. shm_unlink clears any leftover with this name. + shm_unlink(create_name.c_str()); + void *p = open_and_map_shm(create_name, directory_size, ShmAccess::Create, hugepages); + if (p == nullptr) { + std::scoped_lock lk{g_table_mutex}; + release_reserved_slot(idx); + return nullptr; + } + + Note("cache shm: created stripe %s (%zu bytes) for key=%s", create_name.c_str(), directory_size, stripe_key); + return claim_mapped_stripe(idx, p, directory_size); +} + +void +CacheShm::finalize_attach() +{ + if (g_control == nullptr) { + return; + } + + std::scoped_lock lk{g_table_mutex}; + + // With zero claims this run we cannot distinguish "genuinely empty cache" from + // "init aborted" (e.g. a transient volume.config error), so leave every segment + // intact rather than risk reclaiming a valid cache. + if (g_claims_this_run == 0) { + Dbg(dbg_ctl, "finalize_attach: no stripes claimed this run; leaving %u segment(s) intact", g_control->stripe_count); + return; + } + + uint32_t reclaimed = 0; + for (uint32_t i = 0; i < g_control->stripe_count && i < MAX_STRIPES; ++i) { + StripeEntry &e = g_control->stripes[i]; + if (e.shm_name[0] == '\0' || g_entry_claimed[i]) { + continue; // already empty, or claimed by a live stripe this run + } + // Unclaimed, non-empty entry: its stripe left the cache (span dropped, or disk + // failed to open). Unlink the orphan and tombstone the slot for reuse. + std::string name = read_shm_name(e.shm_name); + Note("cache shm: reclaiming orphaned stripe segment %s", name.c_str()); + shm_unlink(name.c_str()); + release_reserved_slot(i); + ++reclaimed; + } + if (reclaimed > 0) { + Note("cache shm: reclaimed %u orphaned stripe segment(s) after attach", reclaimed); + } + + // Trim trailing tombstones so stripe_count tracks the live high-water mark; + // interior tombstones stay (reused by attach_or_create_stripe). + uint32_t live_count = 0; + for (uint32_t i = 0; i < g_control->stripe_count && i < MAX_STRIPES; ++i) { + if (g_control->stripes[i].shm_name[0] != '\0') { + live_count = i + 1; + } + } + const bool count_changed = live_count != g_control->stripe_count; + if (count_changed) { + Note("cache shm: trimming stripe_count %u -> %u after reclaim", g_control->stripe_count, live_count); + g_control->stripe_count = live_count; + } + + if (reclaimed > 0 || count_changed) { + msync(g_control, CONTROL_SIZE, MS_SYNC); + } +} + +bool +CacheShm::is_shm_pointer(char *raw_dir) +{ + if (raw_dir == nullptr) { + return false; + } + std::scoped_lock lk{g_pointers_mutex}; + return g_pointers.find(raw_dir) != g_pointers.end(); +} + +void +CacheShm::mark_clean_shutdown() +{ + if (g_control == nullptr) { + return; + } + Note("cache shm: marking clean shutdown"); + g_control->clean_shutdown = 1; + // Clear owner_pid so the next start's liveness backstop does not mistake our + // (exiting) PID for a live owner. The flock is still held until exit, so a + // concurrent starter is still correctly refused during the shutdown window. + g_control->owner_pid = 0; + msync(g_control, CONTROL_SIZE, MS_SYNC); +} + +void +CacheShm::invalidate_stripe_directory(char *raw_dir) +{ + uint32_t idx = MAX_STRIPES; + { + std::scoped_lock lk{g_pointers_mutex}; + auto it = g_pointers.find(raw_dir); + if (it == g_pointers.end()) { + return; // not a shm-backed dir + } + idx = it->second.index; + } + + // Marked in the control segment, never in the stripe's own header: that header aliases raw_dir, which both the shutdown + // pwrite and the periodic dir sync copy to disk, so a mark there can clear the stripe on the next start instead of + // rebuilding it. Next start, attach_or_create_stripe refuses the entry and creates a fresh segment, whose zero magic + // sends Stripe::init down the disk-read path. + std::scoped_lock lk{g_table_mutex}; + if (g_control == nullptr || idx >= MAX_STRIPES) { + return; + } + g_control->stripes[idx].dir_untrusted = 1; + msync(g_control, CONTROL_SIZE, MS_SYNC); +} + +void +CacheShm::detach_stripe(char *raw_dir) +{ + if (raw_dir == nullptr) { + return; + } + std::scoped_lock lk{g_pointers_mutex}; + auto it = g_pointers.find(raw_dir); + if (it == g_pointers.end()) { + return; + } + // munmap the recorded span; never shm_unlink -- the segment must survive for the + // next start to attach. + munmap(it->first, it->second.size); + g_pointers.erase(it); +} + +#endif // TS_USE_CACHE_SHM diff --git a/src/iocore/cache/CacheShm.h b/src/iocore/cache/CacheShm.h new file mode 100644 index 00000000000..b9bb579f1fd --- /dev/null +++ b/src/iocore/cache/CacheShm.h @@ -0,0 +1,86 @@ +/** @file + + Shared-memory-backed cache directory for fast restart. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include + +struct Store; + +/// Hosts Stripe::Directory::raw_dir in POSIX shared memory so the next start can attach it instead of rebuilding from +/// disk. Purely an optimization: anything wrong drops shm and rebuilds. See the shm-fast-restart developer guide. +class CacheShm +{ +public: + static constexpr std::string_view tag{"ATS-SHM-V1"}; + + enum class Mode { + Disabled, ///< shm.enabled=0; behave like today. + AttachExisting, ///< A valid prior control segment exists; stripes attach by identity or create fresh. + CreateFresh, ///< No/invalid prior control - create everything new (cold path). + }; + + /// Must run after the store is read but before any Stripe is built. + static void initialize(const Store &store); + + static Mode + mode() + { + return _mode; + } + + /// Attaches this stripe's prior segment when one of matching size exists, else creates fresh. nullptr means the caller + /// must fall back to the heap path, which is always the case in Disabled. + static char *attach_or_create_stripe(const char *stripe_key, std::size_t directory_size); + + /// Reclaims segments left by stripes no longer in the cache, e.g. a dropped disk. Call once after all stripes init. + /// Idempotent; no-ops when no stripe came up this run, since that cannot be told from an aborted init. + static void finalize_attach(); + + /// Whether a pointer was returned from attach_or_create_stripe (munmap vs ats_free). + static bool is_shm_pointer(char *raw_dir); + + /// Called after sync_cache_dir_on_shutdown; a crash instead leaves the flag clear, which drops the segment next start. + static void mark_clean_shutdown(); + + /// Marks this stripe's control-segment entry so the next start recreates it rather than attaching. Never writes through + /// raw_dir, which is also the source buffer for the on-disk directory write. No-op for a non-shm pointer. + static void invalidate_stripe_directory(char *raw_dir); + + /// Never shm_unlink: the segment must survive for the next start. No-op for a non-shm pointer. + static void detach_stripe(char *raw_dir); + + /// A writer/reader mismatch forces a drop and rebuild. Exposed for unit testing. + static uint64_t abi_hash(); + + /// Informational only, not a trust gate: a storage change keeps the segment and each stripe attaches by its own identity. + static uint64_t storage_signature(const Store &store); + + /// Backs the concurrent-attach owner-liveness guard. Exposed for unit testing. + static bool process_is_alive(int pid); + +private: + static Mode _mode; +}; diff --git a/src/iocore/cache/P_CacheDir.h b/src/iocore/cache/P_CacheDir.h index 0a9150f4832..e0451721b06 100644 --- a/src/iocore/cache/P_CacheDir.h +++ b/src/iocore/cache/P_CacheDir.h @@ -308,12 +308,14 @@ class Directory */ Dir *get_segment(int s) const; - int probe(const CacheKey *, StripeSM *, Dir *, Dir **); - int insert(const CacheKey *key, StripeSM *stripe, Dir *to_part); - int overwrite(const CacheKey *key, StripeSM *stripe, Dir *to_part, Dir *overwrite, bool must_overwrite = true); - int remove(const CacheKey *key, StripeSM *stripe, Dir *del); - void free_entry(Dir *e, int s); - int check(); + int probe(const CacheKey *, StripeSM *, Dir *, Dir **); + int insert(const CacheKey *key, StripeSM *stripe, Dir *to_part); + int overwrite(const CacheKey *key, StripeSM *stripe, Dir *to_part, Dir *overwrite, bool must_overwrite = true); + int remove(const CacheKey *key, StripeSM *stripe, Dir *del); + void free_entry(Dir *e, int s); + int check(); + /// check() for one segment, so a caller already walking segments need not stream the whole directory a second time. + int check_segment(int s); void cleanup(StripeSM *stripe); void clear_range(off_t start, off_t end, StripeSM *stripe); uint64_t entries_used(); diff --git a/src/iocore/cache/Stripe.cc b/src/iocore/cache/Stripe.cc index 373d545ae1e..261b48a202b 100644 --- a/src/iocore/cache/Stripe.cc +++ b/src/iocore/cache/Stripe.cc @@ -24,6 +24,7 @@ #include "P_CacheDisk.h" #include "P_CacheInternal.h" #include "StripeSM.h" +#include "CacheShm.h" #include "tsutil/DbgCtl.h" @@ -153,15 +154,22 @@ Stripe::_init_directory(std::size_t directory_size, int header_size, int footer_ Dbg(dbg_ctl_cache_init, "Stripe %s: allocating %zu directory bytes for a %lld byte volume (%lf%%)", hash_text.get(), directory_size, (long long)this->len, percent(directory_size, this->len)); - if (ats_hugepage_enabled()) { - this->directory.raw_dir = static_cast(ats_alloc_hugepage(directory_size)); - if (this->directory.raw_dir != nullptr) { - this->directory.raw_dir_huge = true; - } - } - if (nullptr == this->directory.raw_dir) { - this->directory.raw_dir = static_cast(ats_memalign(ats_pagesize(), directory_size)); + // Try shared memory first; a successful attach bypasses the MAP_HUGETLB path below + // (tmpfs can't use it -- CacheShm advises THP instead). + this->directory.raw_dir = CacheShm::attach_or_create_stripe(hash_text.get(), directory_size); + if (this->directory.raw_dir != nullptr) { this->directory.raw_dir_huge = false; + } else { + if (ats_hugepage_enabled()) { + this->directory.raw_dir = static_cast(ats_alloc_hugepage(directory_size)); + if (this->directory.raw_dir != nullptr) { + this->directory.raw_dir_huge = true; + } + } + if (nullptr == this->directory.raw_dir) { + this->directory.raw_dir = static_cast(ats_memalign(ats_pagesize(), directory_size)); + this->directory.raw_dir_huge = false; + } } this->directory.raw_dir_size = directory_size; this->directory.dir = reinterpret_cast(this->directory.raw_dir + header_size); @@ -170,6 +178,61 @@ Stripe::_init_directory(std::size_t directory_size, int header_size, int footer_ this->directory.footer = reinterpret_cast(this->directory.raw_dir + footer_offset); } +// Gate the fast-restart attach: magic/version say the segment looks like a directory, but not that it is safe to follow. +// Rationale for every check, and why the passes are fused per segment, is in doc/developer-guide/cache-architecture/ +// shm-fast-restart.en.rst ("Validating a trusted segment"). On failure the caller falls back to the disk read. +bool +Stripe::_shm_directory_is_valid() +{ + if (this->directory.header->sector_size == 0 || this->directory.header->sector_size > STORE_BLOCK_SIZE) { + return false; + } + + if (this->directory.header->phase > 1) { + return false; + } + + const off_t data_lo = this->start; + const off_t data_hi = this->skip + this->len; + + if (this->directory.header->write_pos < data_lo || this->directory.header->write_pos > data_hi || + this->directory.header->last_write_pos < data_lo || this->directory.header->last_write_pos > data_hi || + this->directory.header->agg_pos < data_lo || this->directory.header->agg_pos > data_hi) { + return false; + } + + // Not a clean-shutdown artifact if the write cursor never quiesced. + if (this->directory.header->agg_pos != this->directory.header->write_pos) { + return false; + } + + const int64_t segment_entries = static_cast(this->directory.buckets) * DIR_DEPTH; + + for (int s = 0; s < this->directory.segments; s++) { + if (this->directory.header->freelist[s] >= segment_entries) { + return false; + } + + // dir_prev only holds a link on empty entries; on an in-use one it is tag/phase/head/pinned. + Dir *seg = this->directory.get_segment(s); + for (int64_t i = 0; i < segment_entries; i++) { + Dir *e = dir_in_seg(seg, i); + if (dir_next(e) >= segment_entries) { + return false; + } + if (dir_is_empty(e) && dir_prev(e) >= segment_entries) { + return false; + } + } + + if (!this->directory.check_segment(s)) { + return false; + } + } + + return true; +} + // coverity[exn_spec_violation] - ink_assert aborts (doesn't throw), Dbg is exception-safe Stripe::~Stripe() { @@ -182,12 +245,19 @@ Stripe::~Stripe() ink_assert(this->directory.raw_dir_size > 0); ink_assert(this->directory.raw_dir_size < MAX_STRIPE_SIZE); + // shm-backed directories must outlive the process; never ats_free or poison them. + const bool is_shm = CacheShm::is_shm_pointer(this->directory.raw_dir); + #ifdef DEBUG - // Poison memory before freeing to help detect use-after-free - memset(this->directory.raw_dir, 0xDE, this->directory.raw_dir_size); + if (!is_shm) { + // Poison memory before freeing to help detect use-after-free + memset(this->directory.raw_dir, 0xDE, this->directory.raw_dir_size); + } #endif - if (this->directory.raw_dir_huge) { + if (is_shm) { + CacheShm::detach_stripe(this->directory.raw_dir); + } else if (this->directory.raw_dir_huge) { ats_free_hugepage(this->directory.raw_dir, this->directory.raw_dir_size); } else { ats_free(this->directory.raw_dir); diff --git a/src/iocore/cache/Stripe.h b/src/iocore/cache/Stripe.h index b99b4773fee..a5656c0a0eb 100644 --- a/src/iocore/cache/Stripe.h +++ b/src/iocore/cache/Stripe.h @@ -148,9 +148,10 @@ class Stripe off_t data_blocks{}; AggregateWriteBuffer _write_buffer; - void _clear_init(std::uint32_t hw_sector_size); - void _init_dir(); - bool flush_aggregate_write_buffer(int fd); + void _clear_init(std::uint32_t hw_sector_size); + void _init_dir(); + bool _shm_directory_is_valid(); + [[nodiscard]] bool flush_aggregate_write_buffer(int fd); private: void _init_hash_text(CacheDisk const *disk, off_t blocks, off_t dir_skip); diff --git a/src/iocore/cache/StripeSM.cc b/src/iocore/cache/StripeSM.cc index 4587963968f..e76745fe74b 100644 --- a/src/iocore/cache/StripeSM.cc +++ b/src/iocore/cache/StripeSM.cc @@ -31,6 +31,7 @@ #include "CacheEvacuateDocVC.h" #include "PreservationTable.h" #include "Stripe.h" +#include "CacheShm.h" #include "iocore/cache/CacheDefs.h" #include "CacheVC.h" @@ -178,6 +179,23 @@ StripeSM::init(bool clear) return clear_dir_aio(); } + // shm fast restart: skip the disk read and recover_data(), which would rescan the tail and discard the entries the + // shm copy preserved. See doc/developer-guide/cache-architecture/shm-fast-restart.en.rst. + if (CacheShm::mode() == CacheShm::Mode::AttachExisting && CacheShm::is_shm_pointer(this->directory.raw_dir)) { + if (this->directory.header->magic == STRIPE_MAGIC && this->directory.footer->magic == STRIPE_MAGIC && + CACHE_DB_MAJOR_VERSION_COMPATIBLE <= this->directory.header->version._major && + this->directory.header->version._major <= CACHE_DB_MAJOR_VERSION && this->_shm_directory_is_valid()) { + Note("attaching cached directory from shm for '%s' (fast restart, recovery skipped)", hash_text.get()); + this->sector_size = this->directory.header->sector_size; + this->scan_pos = this->directory.header->write_pos; + this->_preserved_dirs.periodic_scan(this); + this->set_io_not_in_progress(); + SET_HANDLER(&StripeSM::dir_init_done); + return this->dir_init_done(EVENT_IMMEDIATE, nullptr); + } + Note("shm directory invalid for '%s'; falling back to disk read", hash_text.get()); + } + init_info = new StripeInitInfo(); int footerlen = ROUND_TO_STORE_BLOCK(sizeof(StripeHeaderFooter)); off_t footer_offset = this->dirlen() - footerlen; @@ -1326,9 +1344,21 @@ StripeSM::shutdown(EThread *shutdown_thread) SCOPED_MUTEX_LOCK(lock, this->mutex, shutdown_thread); if (DISK_BAD(this->disk)) { - Dbg(dbg_ctl_cache_dir_sync, "Dir %s: ignoring -- bad disk", this->hash_text.get()); + Dbg(dbg_ctl_cache_dir_sync, "Dir %s: bad disk -- invalidating shm copy for disk recovery", this->hash_text.get()); + CacheShm::invalidate_stripe_directory(this->directory.raw_dir); return; } + + bool shm_backed = CacheShm::is_shm_pointer(this->directory.raw_dir); + + // aggWriteDone advances write_pos again once we drop the mutex, so the shm header is not final; fall through to the + // on-disk write instead, which recover_data() reconciles next start. + if (shm_backed && this->is_io_in_progress()) { + Dbg(dbg_ctl_cache_dir_sync, "Dir %s: AIO write in flight -- invalidating shm copy, syncing dir to disk", this->hash_text.get()); + CacheShm::invalidate_stripe_directory(this->directory.raw_dir); + shm_backed = false; + } + size_t dirlen = this->dirlen(); ink_assert(dirlen > 0); // make clang happy - if not > 0 the vol is seriously messed up if (!this->directory.header->dirty && !this->dir_sync_in_progress) { @@ -1342,7 +1372,14 @@ StripeSM::shutdown(EThread *shutdown_thread) // directories have not been inserted for these writes if (!this->_write_buffer.is_empty()) { Dbg(dbg_ctl_cache_dir_sync, "Dir %s: flushing agg buffer first", this->hash_text.get()); - this->flush_aggregate_write_buffer(this->fd); + if (!this->flush_aggregate_write_buffer(this->fd)) { + // Mark rather than lean on the unquiesced cursor the failure leaves behind: the event system is still up, so a later + // aggWriteDone or agg_wrap() can re-equalize agg_pos and write_pos and the gate would let this segment through. + // Still sync: an entry for unwritten data reads as a miss, but skipping loses every insert since the last sync. + Error("Dir %s: aggregation buffer flush failed during shutdown; syncing the directory to disk", this->hash_text.get()); + CacheShm::invalidate_stripe_directory(this->directory.raw_dir); + shm_backed = false; + } } // We already asserted that dirlen > 0. @@ -1354,6 +1391,14 @@ StripeSM::shutdown(EThread *shutdown_thread) this->directory.footer->sync_serial = this->directory.header->sync_serial; CHECK_DIR(d); + + // Pure waste for a shm-backed dir: it is already current in the segment and is attached directly next start. If the + // segment is later dropped, the on-disk A/B copies plus recover_data() reconcile the tail. + if (shm_backed) { + Note("Dir %s: shm-backed, skipping on-disk directory write", this->hash_text.get()); + return; + } + size_t B = this->directory.header->sync_serial & 1; off_t start = this->skip + (B ? dirlen : 0); B = pwrite(this->fd, this->directory.raw_dir, dirlen, start); diff --git a/src/iocore/cache/unit_tests/test_CacheShm.cc b/src/iocore/cache/unit_tests/test_CacheShm.cc new file mode 100644 index 00000000000..c5156feada0 --- /dev/null +++ b/src/iocore/cache/unit_tests/test_CacheShm.cc @@ -0,0 +1,343 @@ +/** @file + + Unit tests for the cache shared-memory trust gates and control-segment layout: the logic deciding whether a prior shm + segment may be attached or must be dropped and rebuilt from disk. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#include "../CacheShm.h" +#include "shared/cache_shm/Layout.h" +#include "shared/cache_shm/Purge.h" + +#include "iocore/cache/Store.h" +#include "tscore/ink_config.h" +#include "tscore/ink_memory.h" + +#include +#include +#include + +#include +#include +#include + +// Required by the shared test harness (main.cc). +int cache_vols = 1; +bool reuse_existing_cache = false; + +namespace +{ + +// The returned Store owns the Span and frees it on destruction. +void +make_store(Store &store, const char *path, int64_t blocks, int64_t offset = 0) +{ + store.extend(1); + auto *span = new Span(); + span->pathname = ats_strdup(path); + span->blocks = blocks; + span->offset = offset; + span->file_pathname = true; + store.spans[0] = span; +} + +} // namespace + +TEST_CASE("CacheShm ABI hash is stable and non-zero", "[cache][shm]") +{ + const uint64_t a = CacheShm::abi_hash(); + const uint64_t b = CacheShm::abi_hash(); + + // Deterministic: the fingerprint is a pure function of compile-time layout. + CHECK(a == b); + // A zero hash would defeat the trust gate (every segment would look matching); + // the FNV-1a seed and the struct sizes guarantee it is non-zero. + CHECK(a != 0); +} + +TEST_CASE("CacheShm storage signature is sensitive to topology", "[cache][shm]") +{ + Store base; + make_store(base, "/cache/disk0", 1000); + + SECTION("identical topology -> identical signature") + { + Store same; + make_store(same, "/cache/disk0", 1000); + CHECK(CacheShm::storage_signature(base) == CacheShm::storage_signature(same)); + } + + SECTION("different path -> different signature") + { + Store other; + make_store(other, "/cache/disk1", 1000); + CHECK(CacheShm::storage_signature(base) != CacheShm::storage_signature(other)); + } + + SECTION("different size -> different signature") + { + Store resized; + make_store(resized, "/cache/disk0", 2000); + CHECK(CacheShm::storage_signature(base) != CacheShm::storage_signature(resized)); + } + + SECTION("different offset -> different signature") + { + Store moved; + make_store(moved, "/cache/disk0", 1000, /*offset=*/512); + CHECK(CacheShm::storage_signature(base) != CacheShm::storage_signature(moved)); + } + + SECTION("an empty store has a stable signature") + { + Store empty0; + Store empty1; + CHECK(CacheShm::storage_signature(empty0) == CacheShm::storage_signature(empty1)); + } +} + +TEST_CASE("CacheShm control header round-trips through a byte buffer", "[cache][shm]") +{ + using cache_shm::CACHE_SHM_MAGIC; + using cache_shm::CACHE_SHM_SCHEMA_VERSION; + using cache_shm::CacheShmControl; + using cache_shm::CONTROL_SIZE; + + // The on-shm size must equal the struct size; tooling (traffic_ctl) maps + // exactly CONTROL_SIZE bytes and reads the struct out of it. + CHECK(CONTROL_SIZE == sizeof(CacheShmControl)); + + CacheShmControl src; + std::memset(&src, 0, sizeof(src)); + std::memcpy(src.magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)); + src.schema_version = CACHE_SHM_SCHEMA_VERSION; + src.abi_hash = 0x0123456789abcdefULL; + src.storage_signature = 0xfedcba9876543210ULL; + src.clean_shutdown = 1; + src.owner_pid = 4242; + src.stripe_count = 2; + std::strncpy(src.stripes[0].shm_name, "/ats-s0", sizeof(src.stripes[0].shm_name) - 1); + src.stripes[0].raw_dir_size = 4096; + src.stripes[0].stripe_key_hash = 0xaaaabbbbccccddddULL; + std::strncpy(src.stripes[1].shm_name, "/ats-s1", sizeof(src.stripes[1].shm_name) - 1); + src.stripes[1].raw_dir_size = 8192; + src.stripes[1].stripe_key_hash = 0x1111222233334444ULL; + + // Serialize to a raw byte buffer and read it back, mimicking shm attach. + unsigned char buf[CONTROL_SIZE]; + std::memcpy(buf, &src, CONTROL_SIZE); + const auto *dst = reinterpret_cast(buf); + + CHECK(std::memcmp(dst->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)) == 0); + CHECK(dst->schema_version == CACHE_SHM_SCHEMA_VERSION); + CHECK(dst->abi_hash == 0x0123456789abcdefULL); + CHECK(dst->storage_signature == 0xfedcba9876543210ULL); + CHECK(dst->clean_shutdown == 1); + CHECK(dst->owner_pid == 4242); + CHECK(dst->stripe_count == 2); + CHECK(std::string(dst->stripes[0].shm_name) == "/ats-s0"); + CHECK(dst->stripes[0].raw_dir_size == 4096); + CHECK(dst->stripes[0].stripe_key_hash == 0xaaaabbbbccccddddULL); + CHECK(std::string(dst->stripes[1].shm_name) == "/ats-s1"); + CHECK(dst->stripes[1].raw_dir_size == 8192); + CHECK(dst->stripes[1].stripe_key_hash == 0x1111222233334444ULL); +} + +TEST_CASE("CacheShm names respect the macOS PSHMNAMLEN limit", "[cache][shm]") +{ + using cache_shm::MAX_SHM_NAME_LEN; + using cache_shm::StripeEntry; + + // macOS caps POSIX shm names at 31 chars including the leading '/'. The shared + // limit must match so the same naming works on Linux and macOS alike. + CHECK(MAX_SHM_NAME_LEN == 31); + + // The per-stripe name field must hold a maximum-length name plus its NUL. + CHECK(sizeof(StripeEntry{}.shm_name) > MAX_SHM_NAME_LEN); + + // The default control segment name fits comfortably under the limit. + const std::string control_name = cache_shm::control_segment_name("/ats-"); + CHECK(control_name.size() < MAX_SHM_NAME_LEN); +} + +TEST_CASE("CacheShm normalizes the configured name prefix", "[cache][shm]") +{ + using cache_shm::normalize_name_prefix; + + // The operator configures only the middle word; the framing '/' and '-' are + // supplied by the code so a name like "/ats-" cannot be mis-typed. + CHECK(normalize_name_prefix("ats") == "/ats-"); + CHECK(normalize_name_prefix("foo") == "/foo-"); + + // Forgiving of stray framing an operator may carry over (e.g. a pre-existing + // "/ats-" config), so migration cannot produce "//ats--". + CHECK(normalize_name_prefix("/ats-") == "/ats-"); + CHECK(normalize_name_prefix("/ats") == "/ats-"); + CHECK(normalize_name_prefix("ats-") == "/ats-"); + CHECK(normalize_name_prefix("//ats--") == "/ats-"); + + // An embedded '-' in the middle is preserved -- only the framing is trimmed. + CHECK(normalize_name_prefix("ats-v2") == "/ats-v2-"); + + // An embedded '/' is stripped: POSIX shm names permit only the leading '/', so a + // mistyped middle word must not build a name shm_open would reject with EINVAL. + CHECK(normalize_name_prefix("foo/bar") == "/foobar-"); + CHECK(normalize_name_prefix("/ats/v2/") == "/atsv2-"); + CHECK(normalize_name_prefix("a/b/c") == "/abc-"); +} + +TEST_CASE("CacheShm process liveness check backs the concurrent-attach guard", "[cache][shm]") +{ + // Our own PID is, by definition, live -- this is the "a different live owner + // still holds the segment" case the guard refuses to attach over. + CHECK(CacheShm::process_is_alive(static_cast(getpid()))); + + // A zero / negative owner_pid means "no owner recorded" (e.g. after a clean + // shutdown); it must never read as live or the guard would wrongly refuse. + CHECK_FALSE(CacheShm::process_is_alive(0)); + CHECK_FALSE(CacheShm::process_is_alive(-1)); + + // A PID at the top of the range is overwhelmingly unlikely to name a live + // process; kill(pid, 0) returns ESRCH, so it reads as not-alive (a stale + // owner left by a crash, which the guard is free to reclaim). + CHECK_FALSE(CacheShm::process_is_alive(std::numeric_limits::max())); +} + +// The rest of this file needs real shm objects, unlike the layout/fingerprint cases +// above, so it is gated the same way the feature is. +#if TS_USE_CACHE_SHM + +namespace +{ + +// A prefix of our own so these tests can never touch a real instance's segments. +constexpr const char *PURGE_PREFIX_WORD = "atspurgetest"; + +// Valid magic, no owner, one claimed stripe. False if shm is unavailable here. +bool +plant_control_segment(const std::string &prefix, std::size_t size) +{ + const std::string name = cache_shm::control_segment_name(prefix); + shm_unlink(name.c_str()); + int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd < 0) { + return false; + } + if (ftruncate(fd, static_cast(size)) < 0) { + close(fd); + shm_unlink(name.c_str()); + return false; + } + void *addr = mmap(nullptr, cache_shm::CONTROL_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); + if (addr == MAP_FAILED) { + shm_unlink(name.c_str()); + return false; + } + auto *ctrl = static_cast(addr); + std::memset(ctrl, 0, cache_shm::CONTROL_SIZE); + std::memcpy(ctrl->magic, cache_shm::CACHE_SHM_MAGIC, sizeof(cache_shm::CACHE_SHM_MAGIC)); + ctrl->schema_version = cache_shm::CACHE_SHM_SCHEMA_VERSION; + ctrl->owner_pid = 0; // no live owner, so purge is allowed to proceed + ctrl->stripe_count = 1; + // Deliberately unnamed: a foreign build's stripes[] may have another stride, so a correct purge cannot read this. + munmap(addr, cache_shm::CONTROL_SIZE); + return true; +} + +bool +plant_stripe_segment(const std::string &name) +{ + shm_unlink(name.c_str()); + int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd < 0) { + return false; + } + bool ok = ftruncate(fd, 4096) == 0; + close(fd); + return ok; +} + +bool +segment_exists(const std::string &name) +{ + int fd = shm_open(name.c_str(), O_RDONLY, 0); + if (fd < 0) { + return false; + } + close(fd); + return true; +} + +} // namespace + +// A foreign sizeof(CacheShmControl) leaves the frozen header readable but the stripe table not. Purging must fall back to +// the name space, or every stripe segment leaks while `traffic_ctl cache shm clear` reports success. +TEST_CASE("CacheShm purge sweeps by name when the control layout is foreign", "[cache][shm]") +{ + const std::string prefix = cache_shm::normalize_name_prefix(PURGE_PREFIX_WORD); + const std::string stripe_name = cache_shm::stripe_segment_name(prefix, 0); + + // Stands in for a build with a larger stripe table. + if (!plant_control_segment(prefix, cache_shm::CONTROL_SIZE * 2)) { + WARN("shm unavailable in this environment; skipping"); + return; + } + REQUIRE(plant_stripe_segment(stripe_name)); + + const cache_shm::PurgeReport report = cache_shm::purge_segments(prefix); + + CHECK(report.outcome == cache_shm::PurgeOutcome::Purged); + CHECK(report.table_untrusted); + CHECK_FALSE(segment_exists(stripe_name)); + CHECK_FALSE(segment_exists(cache_shm::control_segment_name(prefix))); + + shm_unlink(stripe_name.c_str()); + shm_unlink(cache_shm::control_segment_name(prefix).c_str()); +} + +// The same-size case must walk the table rather than sweep, so a shared name space is not over-swept. +TEST_CASE("CacheShm purge walks the table when the control layout is ours", "[cache][shm]") +{ + const std::string prefix = cache_shm::normalize_name_prefix(PURGE_PREFIX_WORD); + + if (!plant_control_segment(prefix, cache_shm::CONTROL_SIZE)) { + WARN("shm unavailable in this environment; skipping"); + return; + } + + // Absent from the table, so a sweep removes it and a table walk does not. Without this the assertion is vacuous: the + // sweep only records successes, so an empty table yields an identical report either way. + const std::string stray = cache_shm::stripe_segment_name(prefix, 0); + REQUIRE(plant_stripe_segment(stray)); + + const cache_shm::PurgeReport report = cache_shm::purge_segments(prefix); + + CHECK(report.outcome == cache_shm::PurgeOutcome::Purged); + CHECK_FALSE(report.table_untrusted); + CHECK(segment_exists(stray)); + + shm_unlink(stray.c_str()); + shm_unlink(cache_shm::control_segment_name(prefix).c_str()); +} + +#endif // TS_USE_CACHE_SHM diff --git a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc new file mode 100644 index 00000000000..b79d9dea89f --- /dev/null +++ b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc @@ -0,0 +1,417 @@ +/** @file + + Unit tests for StripeSM::shutdown with a shm-backed directory. These need shm actually enabled, which is a process-wide + mode, so they live in their own binary rather than alongside the heap-backed stripe tests. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" +#include "test_doubles.h" + +#include "../CacheShm.h" +#include "shared/cache_shm/Layout.h" +#include "shared/cache_shm/Purge.h" +#include "../P_CacheInternal.h" + +#include "iocore/cache/Store.h" +#include "records/RecCore.h" + +#include +#include + +#include +#include +#include + +// Required by main.h +int cache_vols = 1; +bool reuse_existing_cache = false; + +namespace +{ + +// Our own prefix so these can never touch a real instance's segments, short enough to stay under the 31-char POSIX limit. +constexpr const char *TEST_PREFIX_WORD = "atsunittest"; + +// False when shm is unavailable here, e.g. a sandbox forbidding shm_open, so the test skips rather than fails. +bool +enable_shm() +{ + REQUIRE(RecSetRecordInt("proxy.config.cache.shm.enabled", 1, REC_SOURCE_EXPLICIT) == REC_ERR_OKAY); + REQUIRE(RecSetRecordString("proxy.config.cache.shm.name_prefix", TEST_PREFIX_WORD, REC_SOURCE_EXPLICIT) == REC_ERR_OKAY); + + Store store; + CacheShm::initialize(store); + return CacheShm::mode() != CacheShm::Mode::Disabled; +} + +// shm_unlink only removes the name and live mappings stay valid, so this is safe while a stripe still holds one. +void +unlink_test_segments() +{ + const std::string prefix = cache_shm::normalize_name_prefix(TEST_PREFIX_WORD); + shm_unlink(cache_shm::control_segment_name(prefix).c_str()); + for (uint32_t i = 0; i < 4; ++i) { + shm_unlink(cache_shm::stripe_segment_name(prefix, i).c_str()); + } +} + +// The trust mark lives in the control segment, so shutdown can never reach it through raw_dir. Reads it back the way +// traffic_ctl does, out of a second mapping of the same object. +bool +stripe_marked_untrusted(uint32_t idx) +{ + const std::string prefix = cache_shm::normalize_name_prefix(TEST_PREFIX_WORD); + int fd = shm_open(cache_shm::control_segment_name(prefix).c_str(), O_RDONLY, 0600); + REQUIRE(fd >= 0); + void *addr = mmap(nullptr, cache_shm::CONTROL_SIZE, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + REQUIRE(addr != MAP_FAILED); + auto *ctrl = static_cast(addr); + const bool marked = idx < ctrl->stripe_count && ctrl->stripes[idx].dir_untrusted != 0; + munmap(addr, cache_shm::CONTROL_SIZE); + return marked; +} + +// Refusing a marked entry must reuse its slot, or stripe_count creeps toward MAX_STRIPES across restarts. +uint32_t +control_stripe_count() +{ + const std::string prefix = cache_shm::normalize_name_prefix(TEST_PREFIX_WORD); + int fd = shm_open(cache_shm::control_segment_name(prefix).c_str(), O_RDONLY, 0600); + REQUIRE(fd >= 0); + void *addr = mmap(nullptr, cache_shm::CONTROL_SIZE, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + REQUIRE(addr != MAP_FAILED); + const uint32_t count = static_cast(addr)->stripe_count; + munmap(addr, cache_shm::CONTROL_SIZE); + return count; +} + +// Reaches the protected gate StripeSM::init() consults before fast-attaching. +struct GateStripe : public StripeSM { + using StripeSM::StripeSM; + bool + shm_directory_is_valid() + { + return this->_shm_directory_is_valid(); + } +}; + +// Reaches the protected aggregation buffer so a shutdown flush failure can be staged. +struct AggStripe : public StripeSM { + using StripeSM::StripeSM; + void + stage_pending_bytes(int nbytes) + { + this->_write_buffer.seek(nbytes); + } +}; + +} // namespace + +// The mark that drops a stripe next start must not be written through raw_dir: directory.header aliases it, and both the +// shutdown pwrite and the periodic dir sync copy that buffer to disk, so a mark there clears the stripe instead. +TEST_CASE("StripeSM::shutdown marks the stripe untrusted without touching the directory", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + StripeSM stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + + auto *file{attach_tmpfile_to_stripe(stripe)}; + + // The directory must really live in shm, or the invalidation under test is a no-op. + REQUIRE(CacheShm::is_shm_pointer(stripe.directory.raw_dir)); + + stripe.clear_dir(); + REQUIRE(stripe.directory.header->magic == STRIPE_MAGIC); + + // Together these take the invalidate-then-still-write path. + stripe.io.aiocb.aio_fildes = stripe.fd; + stripe.directory.header->dirty = 1; + + { + SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread()); + stripe.shutdown(this_ethread()); + } + + // The mark goes to the control segment; raw_dir -- which is also the pwrite source -- must come through untouched, so + // the disk copy stays loadable. shutdown() leaves the stripe locked, so read the file unlocked. + CHECK(stripe_marked_untrusted(0)); + CHECK(stripe.directory.header->magic == STRIPE_MAGIC); + + StripeHeaderFooter on_disk{}; + std::size_t headers_read{}; + const uint32_t sync_serial = stripe.directory.footer->sync_serial; + fseek(file, stripe.skip + ((sync_serial & 1) ? stripe.dirlen() : 0), SEEK_SET); + headers_read = fread(&on_disk, sizeof(on_disk), 1, file); + REQUIRE(1 == headers_read); + + CHECK(STRIPE_MAGIC == on_disk.magic); + CHECK(sync_serial == on_disk.sync_serial); + } + + unlink_test_segments(); +} + +// The mark only pays off if the next start honours it. A clean reuse runs first so a fresh segment in the second half +// cannot be mistaken for the control segment having been dropped for some unrelated reason. +TEST_CASE("An untrusted stripe entry is recreated instead of attached", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + constexpr std::size_t dir_size = 8192; + constexpr const char *key = " 0:99"; + constexpr char sentinel = 0x5a; + + char *first = CacheShm::attach_or_create_stripe(key, dir_size); + REQUIRE(first != nullptr); + first[0] = sentinel; + CacheShm::mark_clean_shutdown(); + CacheShm::detach_stripe(first); + const uint32_t count_before = control_stripe_count(); + + // Baseline: an unmarked entry is reused, so the sentinel survives the restart. + REQUIRE(enable_shm()); + REQUIRE(CacheShm::mode() == CacheShm::Mode::AttachExisting); + char *reattached = CacheShm::attach_or_create_stripe(key, dir_size); + REQUIRE(reattached != nullptr); + REQUIRE(reattached[0] == sentinel); + + reattached[0] = sentinel; + CacheShm::invalidate_stripe_directory(reattached); + CacheShm::mark_clean_shutdown(); + CacheShm::detach_stripe(reattached); + + // Same key, same size, entry still present -- but marked, so a fresh (zero-filled) segment must come back instead. + REQUIRE(enable_shm()); + REQUIRE(CacheShm::mode() == CacheShm::Mode::AttachExisting); + char *fresh = CacheShm::attach_or_create_stripe(key, dir_size); + REQUIRE(fresh != nullptr); + CHECK(fresh[0] == 0); + // Refusing the entry must reuse its slot, not orphan it for finalize_attach: a run that never reaches finalize would + // otherwise leak a slot per restart, and both segments would be mapped at once. + CHECK(control_stripe_count() == count_before); + CHECK(!stripe_marked_untrusted(0)); + CacheShm::detach_stripe(fresh); + + unlink_test_segments(); +} + +// The other invalidate caller, and the only one that returns before the directory write: a bad disk must still be marked, +// but nothing may be pushed to a disk we already gave up on. +TEST_CASE("StripeSM::shutdown marks a bad disk's stripe without writing to it", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + StripeSM stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + + auto *file{attach_tmpfile_to_stripe(stripe)}; + REQUIRE(CacheShm::is_shm_pointer(stripe.directory.raw_dir)); + + disk.hw_sector_size = 512; + stripe.clear_dir(); + stripe.directory.header->dirty = 1; + const uint32_t serial_before = stripe.directory.header->sync_serial; + + SET_DISK_BAD((&disk)); + { + SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread()); + stripe.shutdown(this_ethread()); + } + + CHECK(stripe_marked_untrusted(0)); + CHECK(stripe.directory.header->magic == STRIPE_MAGIC); + // Returned before the sync, so neither the serial nor the untouched A/B slot moved. + CHECK(stripe.directory.header->sync_serial == serial_before); + + StripeHeaderFooter on_disk{}; + std::size_t headers_read{}; + fseek(file, stripe.skip + (((serial_before + 1) & 1) ? stripe.dirlen() : 0), SEEK_SET); + headers_read = fread(&on_disk, sizeof(on_disk), 1, file); + // Short read means the slot the sync would have used was never written at all, which is the point. + CHECK((headers_read == 0 || on_disk.sync_serial != serial_before + 1)); + } + + unlink_test_segments(); +} + +// dir_prev carries tag/phase/head/pinned on an in-use entry, so bounds-checking it there compares flag bits against the +// entry count. This stripe has 4 entries per segment against a head bit of 8192, so a healthy directory looked corrupt. +TEST_CASE("A small stripe's directory validates with in-use entries", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + attach_tmpfile_to_stripe(stripe); + // clear_dir() seeds header->sector_size from the disk, and the validator rejects 0. + disk.hw_sector_size = 512; + stripe.clear_dir(); + + REQUIRE(static_cast(stripe.directory.buckets) * DIR_DEPTH < (1 << 13)); + REQUIRE(stripe.shm_directory_is_valid()); + + // Flagged the way a real first fragment is. + Dir *e = stripe.directory.dir; + dir_set_offset(e, 1); + dir_set_head(e, 1); + dir_set_tag(e, 0xfff); + REQUIRE(dir_prev(e) > static_cast(stripe.directory.buckets) * DIR_DEPTH); + + CHECK(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + +// What the flush-failure path actually leans on. A real short write leaves write_pos in range with agg_pos ahead of it, so +// the quiesced-cursor check has to be what rejects the segment -- not the range check, which only a synthetic write_pos +// trips. +TEST_CASE("An unquiesced write cursor fails the attach gate on its own", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + disk.hw_sector_size = 512; + attach_tmpfile_to_stripe(stripe); + stripe.clear_dir(); + REQUIRE(stripe.shm_directory_is_valid()); + + // Every cursor stays inside [start, skip + len], so only agg_pos != write_pos can be the reason. + stripe.directory.header->agg_pos = stripe.directory.header->write_pos + 512; + REQUIRE(stripe.directory.header->agg_pos <= stripe.skip + stripe.len); + REQUIRE(stripe.directory.header->write_pos >= stripe.start); + + CHECK_FALSE(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + +// A failed shutdown flush must still write the on-disk directory -- skipping it drops every insert since the last periodic +// sync -- and must leave the in-segment magic alone, since agg_pos != write_pos already fails the next start's attach gate. +TEST_CASE("StripeSM::shutdown syncs the directory when the agg flush fails", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + AggStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + + auto *file{attach_tmpfile_to_stripe(stripe)}; + REQUIRE(CacheShm::is_shm_pointer(stripe.directory.raw_dir)); + + disk.hw_sector_size = 512; + stripe.clear_dir(); + REQUIRE(stripe.directory.header->magic == STRIPE_MAGIC); + + // Only the flush must fail, so keep the AIO path out of it. + stripe.set_io_not_in_progress(); + stripe.directory.header->dirty = 1; + + // A negative write_pos fails only the aggregation pwrite (EINVAL); the directory write is addressed from skip, so it + // still lands. Breaking the shared fd instead would fail both. + stripe.stage_pending_bytes(512); + stripe.directory.header->write_pos = -1; + + const off_t write_pos_before = stripe.directory.header->write_pos; + const uint32_t serial_before = stripe.directory.header->sync_serial; + + { + SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread()); + stripe.shutdown(this_ethread()); + } + + // Marked like the other two paths that cannot vouch for the directory; the cursor it leaves unquiesced is not durable. + CHECK(stripe.directory.header->write_pos == write_pos_before); + CHECK(stripe.directory.header->agg_pos != stripe.directory.header->write_pos); + CHECK(stripe.directory.header->magic == STRIPE_MAGIC); + CHECK(stripe_marked_untrusted(0)); + + // Pin the slot to the bumped serial: the early return this used to take left the serial alone, so reading the slot it + // implies would just find what clear_dir() wrote and prove nothing. + const uint32_t expect_serial = serial_before + 1; + REQUIRE(stripe.directory.header->sync_serial == expect_serial); + + StripeHeaderFooter on_disk{}; + std::size_t headers_read{}; + fseek(file, stripe.skip + ((expect_serial & 1) ? stripe.dirlen() : 0), SEEK_SET); + headers_read = fread(&on_disk, sizeof(on_disk), 1, file); + REQUIRE(1 == headers_read); + + CHECK(STRIPE_MAGIC == on_disk.magic); + CHECK(expect_serial == on_disk.sync_serial); + } + + unlink_test_segments(); +} diff --git a/src/iocore/cache/unit_tests/test_Stripe.cc b/src/iocore/cache/unit_tests/test_Stripe.cc index 5306ac6ee4d..527239776be 100644 --- a/src/iocore/cache/unit_tests/test_Stripe.cc +++ b/src/iocore/cache/unit_tests/test_Stripe.cc @@ -81,38 +81,6 @@ std::array add_writer_branch_test_cases = { } }; -static void -init_disk(CacheDisk &disk) -{ - disk.path = static_cast(ats_malloc(1)); - disk.path[0] = '\0'; - disk.disk_stripes = static_cast(ats_malloc(sizeof(DiskStripe *))); - disk.disk_stripes[0] = nullptr; - disk.header = static_cast(ats_malloc(sizeof(DiskHeader))); - disk.header->num_volumes = 0; -} - -/* Catch test helper to provide a StripeSM with a valid file descriptor. - * - * The file will be deleted automatically when the application ends normally. - * If the StripeSM already has a valid file descriptor, that file will NOT be - * closed. - * - * @param stripe: A StripeSM object with no valid file descriptor. - * @return The std::FILE* stream if successful, otherwise the Catch test will - * be failed at the point of error. - */ -static std::FILE * -attach_tmpfile_to_stripe(StripeSM &stripe) -{ - auto *file{std::tmpfile()}; - REQUIRE(file != nullptr); - int fd{fileno(file)}; - REQUIRE(fd != -1); - stripe.fd = fd; - return file; -} - // We can't return a stripe from this function because the copy // and move constructors are deleted. static std::FILE * diff --git a/src/iocore/cache/unit_tests/test_doubles.h b/src/iocore/cache/unit_tests/test_doubles.h index e8ce779e3b1..f105b70333c 100644 --- a/src/iocore/cache/unit_tests/test_doubles.h +++ b/src/iocore/cache/unit_tests/test_doubles.h @@ -34,6 +34,7 @@ #endif #include +#include #include #include @@ -137,3 +138,37 @@ class WaitingVC final : public FakeVC EventNotify _notifier; bool _got_callback{false}; }; + +/* Minimal CacheDisk suitable for constructing a StripeSM in a unit test. + */ +inline void +init_disk(CacheDisk &disk) +{ + disk.path = static_cast(ats_malloc(1)); + disk.path[0] = '\0'; + disk.disk_stripes = static_cast(ats_malloc(sizeof(DiskStripe *))); + disk.disk_stripes[0] = nullptr; + disk.header = static_cast(ats_malloc(sizeof(DiskHeader))); + disk.header->num_volumes = 0; +} + +/* Catch test helper to provide a StripeSM with a valid file descriptor. + * + * The file will be deleted automatically when the application ends normally. + * If the StripeSM already has a valid file descriptor, that file will NOT be + * closed. + * + * @param stripe: A StripeSM object with no valid file descriptor. + * @return The std::FILE* stream if successful, otherwise the Catch test will + * be failed at the point of error. + */ +inline std::FILE * +attach_tmpfile_to_stripe(StripeSM &stripe) +{ + auto *file{std::tmpfile()}; + REQUIRE(file != nullptr); + int fd{fileno(file)}; + REQUIRE(fd != -1); + stripe.fd = fd; + return file; +} diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 3ad853798e4..f011c0ccd74 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -82,6 +82,15 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.cache.persist_bad_disks", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.cache.shm.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.cache.shm.name_prefix", RECD_STRING, "ats", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.cache.shm.use_hugepages", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.cache.shm.purge_stale_on_start", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", + RECA_NULL} + , {RECT_CONFIG, "proxy.config.cache.default_volumes", RECD_STRING, "", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.output.logfile.name", RECD_STRING, "traffic.out", RECU_RESTART_TS, RR_REQUIRED, RECC_NULL, nullptr, diff --git a/src/traffic_ctl/CMakeLists.txt b/src/traffic_ctl/CMakeLists.txt index c967d42e9c9..2a678ee78c8 100644 --- a/src/traffic_ctl/CMakeLists.txt +++ b/src/traffic_ctl/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable( traffic_ctl traffic_ctl.cc + CacheShmCommand.cc ConvertConfigCommand.cc CtrlCommands.cc CtrlPrinters.cc diff --git a/src/traffic_ctl/CacheShmCommand.cc b/src/traffic_ctl/CacheShmCommand.cc new file mode 100644 index 00000000000..db5fc612f17 --- /dev/null +++ b/src/traffic_ctl/CacheShmCommand.cc @@ -0,0 +1,302 @@ +/** @file + + traffic_ctl command for inspecting and clearing the cache shared-memory + control segment and its associated stripe segments. + + The status and clear operations work by direct shm_open access rather than + JSONRPC, so they function whether traffic_server is running or not. This + is important for debugging crash-leftover segments when no live process + is available to query. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "CacheShmCommand.h" +#include "shared/cache_shm/Layout.h" +#include "shared/cache_shm/Purge.h" +#include "TrafficCtlStatus.h" + +#include "tscore/ink_config.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ + +// Must match the proxy.config.cache.shm.name_prefix default, or this tool inspects a different segment than the server made. +constexpr const char *DEFAULT_PREFIX = "ats"; + +#if TS_USE_CACHE_SHM + +bool +shm_segment_exists(const std::string &name) +{ + int fd = shm_open(name.c_str(), O_RDONLY, 0); + if (fd < 0) { + return false; + } + close(fd); + return true; +} + +std::string +format_size(uint64_t bytes) +{ + char buf[64]; + if (bytes >= (uint64_t{1} << 30)) { + std::snprintf(buf, sizeof(buf), "%.2f GiB", static_cast(bytes) / (uint64_t{1} << 30)); + } else if (bytes >= (uint64_t{1} << 20)) { + std::snprintf(buf, sizeof(buf), "%.2f MiB", static_cast(bytes) / (uint64_t{1} << 20)); + } else if (bytes >= (uint64_t{1} << 10)) { + std::snprintf(buf, sizeof(buf), "%.2f KiB", static_cast(bytes) / (uint64_t{1} << 10)); + } else { + std::snprintf(buf, sizeof(buf), "%llu B", static_cast(bytes)); + } + return buf; +} + +using cache_shm::process_is_alive; +using cache_shm::read_shm_name; + +#endif // TS_USE_CACHE_SHM + +} // namespace + +CacheShmCommand::CacheShmCommand(ts::Arguments *args) : CtrlCommand(args) +{ + if (get_parsed_arguments()->get(STATUS_STR)) { + _invoked_func = [this]() { status(); }; + } else if (get_parsed_arguments()->get(CLEAR_STR)) { + _invoked_func = [this]() { clear(); }; + } +} + +std::string +CacheShmCommand::get_prefix() +{ + // Framed the same way the server does, so the two agree on segment names. + std::string configured = DEFAULT_PREFIX; + if (auto arg = get_parsed_arguments()->get(PREFIX_STR); arg && !arg.empty()) { + configured = arg.value(); + } + return cache_shm::normalize_name_prefix(configured); +} + +// A miss against the default prefix usually means name_prefix is set and was not repeated here; say so rather than leave +// the operator concluding shm was never enabled. +std::string +CacheShmCommand::default_prefix_hint() +{ + if (auto arg = get_parsed_arguments()->get(PREFIX_STR); arg && !arg.empty()) { + return {}; + } + return " (if proxy.config.cache.shm.name_prefix is set, pass --prefix )"; +} + +void +CacheShmCommand::report_unsupported() +{ + std::cerr << "cache shm: this build has no POSIX shared memory support, so the cache shm " + "fast-restart feature is compiled out; there is nothing to inspect or clear.\n"; + App_Exit_Status_Code = CTRL_EX_UNIMPLEMENTED; +} + +void +CacheShmCommand::status() +{ +#if !TS_USE_CACHE_SHM + report_unsupported(); +#else + const std::string prefix = get_prefix(); + const std::string control_name = cache_shm::control_segment_name(prefix); + + int fd = shm_open(control_name.c_str(), O_RDONLY, 0); + if (fd < 0) { + std::cerr << "cache shm: control segment '" << control_name << "' not found: " << std::strerror(errno) << default_prefix_hint() + << '\n'; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + } + + struct stat sb { + }; + if (fstat(fd, &sb) < 0) { + std::cerr << "cache shm: fstat failed: " << std::strerror(errno) << '\n'; + close(fd); + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + } + + if (!cache_shm::is_own_control_size(static_cast(sb.st_size))) { + // stripes[] may have another stride, so interpreting it with our layout would print nonsense. `clear` refuses the + // same segment for the same reason. + std::cerr << "cache shm: control segment '" << control_name << "' is " << sb.st_size << " bytes, not this build's " + << cache_shm::CONTROL_SIZE << "; it was written by a different build and cannot be interpreted.\n"; + close(fd); + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + } + + void *addr = mmap(nullptr, cache_shm::CONTROL_SIZE, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + if (addr == MAP_FAILED) { + std::cerr << "cache shm: mmap failed: " << std::strerror(errno) << '\n'; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + } + + const auto *ctrl = static_cast(addr); + + const bool magic_ok = std::memcmp(ctrl->magic, cache_shm::CACHE_SHM_MAGIC, sizeof(cache_shm::CACHE_SHM_MAGIC)) == 0; + const bool schema_ok = ctrl->schema_version == cache_shm::CACHE_SHM_SCHEMA_VERSION; + + std::cout << "Control segment: " << control_name << '\n'; + std::cout << " segment size: " << sb.st_size << " bytes (" << format_size(sb.st_size) << ")\n"; + std::cout << " magic: "; + for (char c : ctrl->magic) { + if (c >= 0x20 && c < 0x7f) { + std::cout << c; + } + } + std::cout << (magic_ok ? " [valid]" : " [INVALID]") << '\n'; + std::cout << " schema_version: " << ctrl->schema_version << (schema_ok ? " [valid]" : " [INVALID]") << '\n'; + std::cout << " abi_hash: 0x" << std::hex << ctrl->abi_hash << std::dec << '\n'; + std::cout << " storage_sig: 0x" << std::hex << ctrl->storage_signature << std::dec << '\n'; + std::cout << " clean_shutdown: " << static_cast(ctrl->clean_shutdown) + << (ctrl->clean_shutdown ? " (clean)" : " (DIRTY -- next start will rebuild)") << '\n'; + std::cout << " owner_pid: " << ctrl->owner_pid; + if (ctrl->owner_pid == 0) { + std::cout << " (none -- not currently attached)"; + } else if (process_is_alive(ctrl->owner_pid)) { + std::cout << " (LIVE -- a running traffic_server owns this segment)"; + } else { + std::cout << " (stale -- owner no longer running)"; + } + std::cout << '\n'; + std::cout << " stripe_count: " << ctrl->stripe_count << '\n'; + + if (!magic_ok || !schema_ok) { + std::cout << "\nHeader is invalid; not interpreting stripe table.\n"; + munmap(addr, cache_shm::CONTROL_SIZE); + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + } + + const uint32_t shown = std::min(ctrl->stripe_count, cache_shm::MAX_STRIPES); + + if (shown > 0) { + std::cout << "\nStripes:\n"; + for (uint32_t i = 0; i < shown; ++i) { + const auto &entry = ctrl->stripes[i]; + std::string name = read_shm_name(entry.shm_name); + if (name.empty()) { + std::cout << " [" << i << "] (tombstone -- slot free for reuse)\n"; + continue; + } + const bool present = shm_segment_exists(name); + std::cout << " [" << i << "] " << name << " size=" << entry.raw_dir_size << " (" << format_size(entry.raw_dir_size) << ") " + << (present ? "present" : "MISSING") << (entry.dir_untrusted ? " untrusted (will rebuild from disk)" : "") << '\n'; + } + } + + if (ctrl->stripe_count > cache_shm::MAX_STRIPES) { + std::cout << "\n(stripe_count " << ctrl->stripe_count << " exceeds MAX_STRIPES " << cache_shm::MAX_STRIPES << "; truncated.)\n"; + } + + munmap(addr, cache_shm::CONTROL_SIZE); +#endif // TS_USE_CACHE_SHM +} + +void +CacheShmCommand::clear() +{ +#if !TS_USE_CACHE_SHM + report_unsupported(); +#else + // Shared with the server's purge-on-disabled-start; this only renders the result and sets the exit code. + const cache_shm::PurgeReport report = cache_shm::purge_segments(get_prefix()); + + switch (report.outcome) { + case cache_shm::PurgeOutcome::BadPrefix: + std::cerr << "cache shm: invalid prefix (must be non-empty and begin with '/').\n"; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + case cache_shm::PurgeOutcome::NotPresent: + std::cerr << "cache shm: control segment '" << report.control_name << "' not found (" << std::strerror(report.sys_errno) << ")" + << default_prefix_hint() << "; nothing to clear.\n"; + std::cout << "Removed 0 segment(s).\n"; + return; + case cache_shm::PurgeOutcome::OpenFailed: + // Not ENOENT: the segment may well exist but we could not open it (e.g. EACCES on a + // segment owned by another user). Report the real errno and fail rather than claim success. + std::cerr << "cache shm: cannot open control segment '" << report.control_name << "' (" << std::strerror(report.sys_errno) + << "); cannot clear.\n"; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + case cache_shm::PurgeOutcome::MapFailed: + std::cerr << "cache shm: mmap failed while reading stripe table: " << std::strerror(report.sys_errno) << '\n'; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + case cache_shm::PurgeOutcome::StatFailed: + std::cerr << "cache shm: cannot stat control segment '" << report.control_name << "' (" << std::strerror(report.sys_errno) + << "); cannot clear.\n"; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + case cache_shm::PurgeOutcome::OwnedByLive: + // Refuse: unlinking a live owner's segments would orphan its fast restart. + std::cerr << "cache shm: control segment '" << report.control_name << "' is owned by a live traffic_server (pid " + << report.owner_pid << "); refusing to clear. Stop traffic_server first.\n"; + App_Exit_Status_Code = CTRL_EX_ERROR; + return; + case cache_shm::PurgeOutcome::TooSmall: + case cache_shm::PurgeOutcome::Purged: + break; + } + + if (report.table_untrusted) { + std::cerr << "cache shm: control segment '" << report.control_name << "' (" << report.segment_size + << " bytes) was written by a different build; its stripe table could not be read, so every '" << get_prefix() + << "s' name was swept instead.\n"; + } + + for (const auto &u : report.unlinked) { + if (u.error == 0) { + std::cout << "unlinked " << u.name << '\n'; + } else if (u.error != ENOENT) { + std::cerr << "failed to unlink " << u.name << ": " << std::strerror(u.error) << '\n'; + } + } + + const unsigned failures = report.failures(); + std::cout << "Removed " << report.removed() << " segment(s)"; + if (failures != 0) { + std::cout << ", " << failures << " failure(s)"; + App_Exit_Status_Code = CTRL_EX_ERROR; + } + std::cout << ".\n"; +#endif // TS_USE_CACHE_SHM +} diff --git a/src/traffic_ctl/CacheShmCommand.h b/src/traffic_ctl/CacheShmCommand.h new file mode 100644 index 00000000000..874a17364e5 --- /dev/null +++ b/src/traffic_ctl/CacheShmCommand.h @@ -0,0 +1,50 @@ +/** @file + + traffic_ctl command for inspecting and clearing the cache shared-memory + control segment and its associated stripe segments. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "CtrlCommands.h" + +#include + +class CacheShmCommand : public CtrlCommand +{ +public: + CacheShmCommand(ts::Arguments *args); + +private: + static inline const std::string STATUS_STR{"status"}; + static inline const std::string CLEAR_STR{"clear"}; + static inline const std::string PREFIX_STR{"prefix"}; + + void status(); + void clear(); + + /// Report that this build has no POSIX shared memory support (see TS_USE_CACHE_SHM), + /// so both subcommands have nothing to act on. + void report_unsupported(); + + std::string get_prefix(); + std::string default_prefix_hint(); +}; diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index f39759e2f2f..d5ba581419b 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -31,6 +31,7 @@ #include "tscore/signals.h" #include "CtrlCommands.h" +#include "CacheShmCommand.h" #include "ConvertConfigCommand.h" #include "FileConfigCommand.h" #include "SSLMultiCertCommand.h" @@ -320,6 +321,20 @@ main([[maybe_unused]] int argc, const char **argv) .add_option("--params", "-p", "Parameters to be passed in the request, YAML or JSON format", "", MORE_THAN_ONE_ARG_N, "", "") .add_example_usage("traffic_ctl rpc invoke foo_bar -p \"numbers: [1, 2, 3]\""); + // cache shm commands - operate directly on POSIX shared memory; no running server required. + auto &shm_command = cache_command.add_command("shm", "Inspect and manage cache shared-memory segments").require_commands(); + // No parser-level default for --prefix: ArgParser injects defaults into the parsed + // arguments, which would make an omitted --prefix indistinguishable from an explicit + // one. CacheShmCommand supplies the runtime default and keys its "did you set + // name_prefix?" hint off the option being absent. + shm_command.add_option("--prefix", "-p", "shm name prefix word, framed as /- (default 'ats')", "", 1, ""); + shm_command.add_command("status", "Show the cache shared-memory control segment and stripe table", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl cache shm status") + .add_example_usage("traffic_ctl cache shm status --prefix ats-t"); + shm_command.add_command("clear", "Unlink the cache shared-memory control and stripe segments", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl cache shm clear") + .add_example_usage("traffic_ctl cache shm clear --prefix ats-t"); + auto create_command = [](ts::Arguments &args) -> std::unique_ptr { if (args.get("config")) { if (args.get("convert")) { @@ -334,8 +349,15 @@ main([[maybe_unused]] int argc, const char **argv) return std::make_unique(&args); } + if (args.get("cache")) { + // `cache shm` reads the shm segments directly, so it must not be routed through the RPC-backed CacheCommand. + if (args.get("shm")) { + return std::make_unique(&args); + } + return std::make_unique(&args); + } + static const std::map(ts::Arguments *)>> factories = { - {"cache", [](ts::Arguments *a) { return std::make_unique(a); } }, {"metric", [](ts::Arguments *a) { return std::make_unique(a); } }, {"server", [](ts::Arguments *a) { return std::make_unique(a); } }, {"storage", [](ts::Arguments *a) { return std::make_unique(a); } }, diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 746eeca499b..91b0677e042 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -151,6 +151,7 @@ produce_features(bool json) print_feature("TS_HAS_PROFILER", TS_HAS_PROFILER, json); print_feature("TS_USE_FAST_SDK", TS_USE_FAST_SDK, json); print_feature("TS_USE_DIAGS", TS_USE_DIAGS, json); + print_feature("TS_USE_CACHE_SHM", TS_USE_CACHE_SHM, json); print_feature("TS_USE_EPOLL", TS_USE_EPOLL, json); print_feature("TS_USE_KQUEUE", TS_USE_KQUEUE, json); print_feature("TS_USE_POSIX_CAP", TS_USE_POSIX_CAP, json); diff --git a/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py b/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py new file mode 100644 index 00000000000..7b7921783be --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py @@ -0,0 +1,230 @@ +''' +Verify per-stripe partial attach when a disk is dropped from storage.yaml +(the "bad disk" case). A storage change no longer cold-starts every stripe: +the stripes on healthy, unchanged disks fast-attach their prior shm segments +while the segment left behind by the removed disk is reclaimed. + +ts1 caches an object across two disks and clean-shuts-down (marking the shm +clean). ts2 starts against the *same* shm prefix but with the second disk +removed from storage.yaml -- simulating a bad disk dropped by the operator. +ts2 must: + - keep the existing control segment (partial attach, not a full recreate), + - fast-attach the surviving disk's stripe by its stable identity, + - reclaim the orphaned stripe segment of the removed disk, + - and still serve traffic. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import uuid + +Test.Summary = ''' +Dropping a disk from storage.yaml fast-attaches the surviving stripes from +shm and reclaims the orphaned stripe segment of the removed disk. +''' +Test.ContinueOnFail = True + + +class CacheShmBadDiskDroppedTest: + """ + A stripe's shm identity is its hash_text -- the disk seed (path or + hash_base_string) plus that disk's own dir_skip:blocks, read from the + disk's persisted header. None of those depend on the other disks, so when + one disk is removed from storage.yaml the surviving disks compute the + same hash_text as before and re-attach their prior shm segments. The + removed disk's stripe is no longer present, so its control entry is never + claimed and finalize_attach() reclaims the orphaned segment. + + ts1 starts cold across disk_a + disk_b, populates the cache, and clean-shuts + down. ts2 starts against disk_a only (disk_b "fails"/dropped) sharing the + shm prefix, and asserts ts2: + - enters partial-attach mode (storage signature changed) keeping the + control segment rather than recreating it, + - fast-attaches the surviving disk_a stripe from shm, + - reclaims exactly the orphaned disk_b stripe segment, + - reports neither an unclean shutdown nor a schema/ABI mismatch, + - and serves a request (200). + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB per disk + + def __init__(self): + self._setup_shared_state() + # ts1 sees both disks; ts2 sees only disk_a (disk_b dropped). + self.ts1 = self._configure_ts('shmbd_ts1', [self._storage_path_a, self._storage_path_b]) + self.ts2 = self._configure_ts('shmbd_ts2', [self._storage_path_a]) + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + # Absolute paths keep the spans independent of MakeATSProcess's + # per-instance STORAGEDIR so disk_a has identical geometry for ts1 and + # ts2 (hence identical stripe identity -> fast attach). + self._storage_path_a = os.path.join(shared_storage_dir, 'disk_a.img') + self._storage_path_b = os.path.join(shared_storage_dir, 'disk_b.img') + for path in (self._storage_path_a, self._storage_path_b): + with open(path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'bd' = bad-disk-dropped variant. + self._shm_prefix = f'/cshmbd-{os.getpid() % 100000}-' + + def _configure_ts(self, name, storage_paths): + ts = Test.MakeATSProcess(name) + storage_lines = ['cache:', ' spans:'] + for i, storage_path in enumerate(storage_paths): + storage_lines += [ + f' - name: disk.{i}', + f' path: {storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ] + storage_lines += [' volumes:', ' - id: 1', ' scheme: http', ' size: 100%'] + ts.Disk.storage_yaml.AddLines(storage_lines) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm|cache_init', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 cold start across both disks, clean shutdown. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'created stripe \S+ \(\d+ bytes\) for key=', 'ts1 should create the shm-backed stripe segments') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + + # ts2 warm start with disk_b dropped: partial attach -- the surviving + # disk_a stripe attaches, the orphaned disk_b segment is reclaimed. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'attaching up to \d+ stripes \(fast restart, partial -- storage changed\)', + 'ts2 must enter partial-attach mode after the disk was dropped') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'attached stripe \S+ \(\d+ bytes\) for key=', 'ts2 must fast-attach the surviving disk_a stripe from shm') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: reclaiming orphaned stripe segment', 'ts2 must reclaim the dropped disk_b stripe segment') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'reclaimed \d+ orphaned stripe segment\(s\) after attach', 'ts2 must report the reclaim summary') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: creating fresh control segment', 'ts2 must keep the control segment across the disk drop') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', + 'the partial attach must be due to the disk drop, not an unclean shutdown') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: (schema|ABI) mismatch', 'the partial attach must be due to the disk drop, not schema/ABI') + + def _populate_cache(self): + tr = Test.AddTestRun('Populate cache via ts1 (disk_a + disk_b)') + tr.Processes.Default.StartBefore(self.ts1) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts1.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', 'ts1 first GET should return 200') + tr.StillRunningAfter = self.ts1 + + def _clean_shutdown_ts1(self): + tr = Test.AddTestRun('Drain and clean-shutdown ts1') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shmbd_ts1 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _dump_shm_state(self): + # Between ts1's clean shutdown and ts2's start the control segment is + # marked clean and records both stripes (nothing reclaimed yet). Capture + # it with `traffic_ctl cache shm status` and compare against a gold file. + # The gold masks the run-specific names, the ABI/storage hashes, and the + # page-rounded sizes with the `` wildcard, so what is asserted literally + # is the meaningful state: valid magic/schema, clean_shutdown=1, + # stripe_count=2, and both stripe segments present. + tr = Test.AddTestRun('Dump shm control state after ts1 clean shutdown') + # Use ts1's Env: it has been started, so the per-instance bin dir is on + # PATH (ts2's Env only gains it once ts2 starts, which is the next step). + # `cache shm status` reads the segment directly and needs no live server. + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm status --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = 'gold/cache_shm_state_after_shutdown.gold' + # The gold's stripe rows end at `present`. A stripe marked untrusted at shutdown appends to that line, so say so + # explicitly -- otherwise a stray mark (e.g. an AIO write still in flight) surfaces as an opaque gold diff. + tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + 'untrusted', 'this cache was shut down cleanly, so no stripe should be marked untrusted') + + def _verify_survivor_attach_and_reclaim(self): + tr = Test.AddTestRun('Start ts2 (disk_b dropped); verify survivor fast-attach + orphan reclaim') + tr.Processes.Default.StartBefore(self.ts2) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts2.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + '200', 'ts2 should serve correctly after the partial attach') + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown_ts2(self): + # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a + # segment a live traffic_server still owns, so the segments must be + # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + tr = Test.AddTestRun('Drain and clean-shutdown ts2') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shmbd_ts2 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + # A clean shutdown deliberately keeps the control + live stripe segments + # for the next fast restart, so they outlive the test. Unlink them by + # prefix to avoid leaking POSIX shm across repeated local runs (macOS has + # no /dev/shm to clear out of band). + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._populate_cache() + self._clean_shutdown_ts1() + self._dump_shm_state() + self._verify_survivor_attach_and_reclaim() + self._clean_shutdown_ts2() + self._cleanup_shm() + + +CacheShmBadDiskDroppedTest().run() diff --git a/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py b/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py new file mode 100644 index 00000000000..90453312708 --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py @@ -0,0 +1,185 @@ +''' +Verify the concurrent-attach guard: a second traffic_server must never map the +shm directory read-write underneath a live owner. ts1 cold-starts and becomes +the owner of the control segment (it sets owner_pid and, on Linux, holds an +exclusive flock for its lifetime). While ts1 is still running, ts2 starts +against the *same* shm prefix; it must refuse shm for this run, disable it, and +come up on its own disk cache without touching ts1's segment. ts1 keeps serving +throughout. + +The two instances use *separate* on-disk cache files so the test isolates the +shm concurrent-attach guard from any contention over a shared cache file. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import uuid + +Test.Summary = ''' +A second traffic_server refuses to attach the shm directory while a live owner +holds it, disabling shm for its run instead of attaching concurrently. +''' +Test.ContinueOnFail = True + + +class CacheShmConcurrentAttachTest: + """ + The concurrent-attach guard (P0). A live owner is still mapping the Dir + read-write; a second writer would corrupt it, and clean_shutdown is no + protection against a concurrent *live* run. The guard fires from either of + two mechanisms, so this test asserts on the shared tail of both messages: + - Linux: ts1 holds an exclusive flock on the control segment for its + lifetime; ts2's lock attempt returns HeldByOther ("... is locked by a + live owner ..."). + - macOS (flock unsupported): the owner_pid liveness backstop fires + instead ("... claims a live owner ..."). + Both end in "disabling shm this run to avoid concurrent attach" and set the + run to shm-disabled, which is what this test pins -- so it runs on every + platform. + + ts2 must: + - log the concurrent-attach refusal, + - NOT create or attach a control segment (it bails before either), + - still serve a request (200) from its own disk cache. + ts1 must keep running and serving the whole time. + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + def __init__(self): + self._setup_shared_state() + # Same shm prefix, different storage files: the collision under test is + # purely on the shm control segment, not the on-disk cache. + self.ts1 = self._configure_ts('shmc_ts1', self._storage_path_a) + self.ts2 = self._configure_ts('shmc_ts2', self._storage_path_b) + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._storage_path_a = os.path.join(shared_storage_dir, 'disk_a.img') + self._storage_path_b = os.path.join(shared_storage_dir, 'disk_b.img') + for path in (self._storage_path_a, self._storage_path_b): + with open(path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'c' = concurrent-attach variant. + self._shm_prefix = f'/cshmc-{os.getpid() % 100000}-' + + def _configure_ts(self, name, storage_path): + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 is the owner: it creates the fresh control segment. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create and own the shm control segment') + + # ts2 starts while ts1 owns the segment: it must refuse and disable shm. + # The message head differs by platform (flock vs owner_pid backstop); the + # tail is common, so anchor on it. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'disabling shm this run to avoid concurrent attach', 'ts2 must refuse to attach while ts1 owns the segment') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: creating fresh control segment', 'ts2 must not create a control segment when it refuses shm') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: attaching up to \d+ stripes \(fast restart', "ts2 must not attach ts1's live control segment") + + def _start_owner(self): + tr = Test.AddTestRun('Cold-start ts1 (becomes the shm owner)') + tr.Processes.Default.StartBefore(self.ts1) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts1.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', 'ts1 GET should return 200') + tr.StillRunningAfter = self.ts1 + + def _start_second_refused(self): + # ts1 is still running (kept alive by StillRunningAfter above), so ts2's + # start hits the concurrent-attach guard. + tr = Test.AddTestRun('Start ts2 while ts1 is live; ts2 must refuse shm and serve from disk') + tr.Processes.Default.StartBefore(self.ts2) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts2.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + '200', 'ts2 should serve from its own disk cache with shm disabled') + tr.StillRunningAfter = self.ts1 + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown(self, ts, name): + tr = Test.AddTestRun(f'Drain and clean-shutdown {name}') + tr.Processes.Default.Env = ts.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} {name} --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + # ts1 (the owner) is stopped before this so clean_shutdown clears + # owner_pid; otherwise `cache shm clear` refuses a live owner. + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._start_owner() + self._start_second_refused() + # Stop the non-owner first, then the owner (clears owner_pid), then clear. + self._clean_shutdown(self.ts2, 'shmc_ts2') + self._clean_shutdown(self.ts1, 'shmc_ts1') + self._cleanup_shm() + + +CacheShmConcurrentAttachTest().run() diff --git a/tests/gold_tests/cache/cache_shm_control_size_mismatch.test.py b/tests/gold_tests/cache/cache_shm_control_size_mismatch.test.py new file mode 100644 index 00000000000..9da1ba298f7 --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_control_size_mismatch.test.py @@ -0,0 +1,202 @@ +''' +Verify a control segment whose size is not this build's sizeof(CacheShmControl) is +dropped and recreated rather than wedging shm off. Such a segment is what an upgrade +that changes the control layout (a MAX_STRIPES bump, a longer shm_name, a new +StripeEntry field) leaves behind: the size-checked attach cannot map it, and without +the frozen-header drop path the O_EXCL create would then fail with EEXIST on every +restart until an operator ran `traffic_ctl cache shm clear`. + +ts1 cold-starts, caches an object, and clean-shuts-down. The control segment file +under /dev/shm is then grown past any size this build could have written, so ts2 sees +a foreign-size segment: it must drop it, create a fresh one, and rebuild from disk. +ts3 then starts against what ts2 left behind and must fast-attach normally, which is +what proves the drop actually healed rather than deferring the wedge by one restart. + +Linux-only: it resizes the /dev/shm segment file, which exists only on Linux (macOS +POSIX shm segments are not path-addressable). +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import platform +import sys +import uuid + +Test.Summary = ''' +A control segment written with a different sizeof(CacheShmControl) is dropped and +recreated, never left to wedge the create path. +''' +Test.ContinueOnFail = True + +# The poke drives the gate by editing /dev/shm directly, which is a Linux facility; +# macOS POSIX shm is not exposed as a file. There is no Condition for the platform, +# so gate with a lambda (ports.py branches on platform the same way). +Test.SkipUnless(Condition(lambda: platform.system() == 'Linux', "shm byte-poke gates need Linux /dev/shm")) + + +class CacheShmControlSizeMismatchTest: + """ + The control-segment size gate. Every field the attach path needs to identify a + segment (magic, schema_version, abi_hash) and to guard dropping it (owner_pid, + clean_shutdown) lives in a frozen prefix, so a segment of any other size is still + readable that far: it is guarded, dropped, and recreated in one start. + + Sequence: ts1 creates a clean segment; the segment file is grown past this build's + size; ts2 must report the size mismatch, drop, recreate, and serve; ts3 must then + fast-attach the segment ts2 created. + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + POKE_SCRIPT = 'shm_poke.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + # Writing one byte at this offset grows the (sparse) segment file well past + # sizeof(CacheShmControl) rounded up to a page, which is what the attach path + # accepts. MAX_CONTROL_SEGMENT_BYTES caps the struct at 32 KiB, so 1 MiB is + # beyond reach of any layout this build could compile. The header itself is left + # intact, so the size is the only thing that makes the segment foreign. + GROW_OFFSET = 1024 * 1024 + GROW_BYTE_HEX = '00' + + def __init__(self): + self._setup_shared_state() + self.ts1 = self._configure_ts('shmz_ts1') + self.ts2 = self._configure_ts('shmz_ts2') + self.ts3 = self._configure_ts('shmz_ts3') + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + Test.Setup.Copy(os.path.join(Test.TestDirectory, self.POKE_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._shared_storage_path = os.path.join(shared_storage_dir, 'disk.img') + with open(self._shared_storage_path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'z' = size-mismatch variant. + # (This test is Linux-only, but keep the prefix short for consistency.) + self._shm_prefix = f'/cshmz-{os.getpid() % 100000}-' + self._control_file = '/dev/shm/' + self._shm_prefix.lstrip('/') + 'control' + + def _configure_ts(self, name): + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {self._shared_storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 cold start, clean shutdown -- a valid, clean segment to resize. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + + # ts2 start against the resized segment: report, drop, recreate, rebuild. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r"cache shm: control segment \S+ is \d+ bytes, not this build's \d+; dropping it", + 'ts2 must report the control segment size mismatch') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts2 must recreate the control segment after the drop') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: failed to create control segment', + 'the drop must leave the name free, so the O_EXCL create cannot fail with EEXIST') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'\(fast restart, recovery skipped\)', 'ts2 must rebuild from disk, never fast-attach a foreign-size segment') + + # ts3 fast-attaches what ts2 created: the drop healed, it did not just defer. + self.ts3.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: attaching up to \d+ stripes \(fast restart', 'ts3 should attach the segment ts2 created') + self.ts3.Disk.diags_log.Content += Testers.ContainsExpression( + r"attaching cached directory from shm for '.+' \(fast restart", 'ts3 should reuse the per-stripe directory from shm') + self.ts3.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: control segment \S+ is \d+ bytes', 'ts3 should see a segment of the expected size') + self.ts3.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: creating fresh control segment', 'ts3 should not have to create another control segment') + + def _get(self, ts, description): + tr = Test.AddTestRun(description) + tr.Processes.Default.StartBefore(ts) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{ts.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', f'{description}: should return 200') + tr.StillRunningAfter = ts + + def _clean_shutdown(self, ts, name): + tr = Test.AddTestRun(f'Drain and clean-shutdown {name}') + tr.Processes.Default.Env = ts.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} {name} --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _grow_control_segment(self): + # ts1 is dead; the segment is just a file now. Extend it without touching the + # header, so only its size makes it foreign to this build. + tr = Test.AddTestRun('Grow the shm control segment past this build\'s size') + tr.Processes.Default.Command = ( + f'{sys.executable} ./{self.POKE_SCRIPT} {self._control_file} {self.GROW_OFFSET} {self.GROW_BYTE_HEX}') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts3.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._get(self.ts1, 'Cold-start ts1 and cache an object') + self._clean_shutdown(self.ts1, 'shmz_ts1') + self._grow_control_segment() + self._get(self.ts2, 'Start ts2; the foreign-size segment is dropped and recreated') + self._clean_shutdown(self.ts2, 'shmz_ts2') + self._get(self.ts3, 'Start ts3; the recreated segment fast-attaches') + self._clean_shutdown(self.ts3, 'shmz_ts3') + self._cleanup_shm() + + +CacheShmControlSizeMismatchTest().run() diff --git a/tests/gold_tests/cache/cache_shm_dir_invalid.test.py b/tests/gold_tests/cache/cache_shm_dir_invalid.test.py new file mode 100644 index 00000000000..108745ba37b --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_dir_invalid.test.py @@ -0,0 +1,237 @@ +''' +Verify the per-stripe shm directory trust gates: an in-shm directory whose header +fields are out of range is rejected and rebuilt from disk, never fast-attached. +These branches (Stripe::_shm_directory_is_valid) are what stand between a stale or +torn shm directory and out-of-bounds disk I/O, so each is driven directly. + +ts1 cold-starts, caches an object, and clean-shuts-down, which leaves the control +segment marked clean and the stripe segment holding a valid directory. The stripe +segment file under /dev/shm is then tampered with between runs: + + * ts2 sees write_pos pushed past the end of the stripe. + * ts3 sees freelist[0] pushed past the segment's entry count. + +Each instance must attach the shm segments, reject the directory, fall back to the +disk read + recover_data(), and still serve the object out of cache. + +Linux-only: it pokes raw bytes in the /dev/shm segment files, which exist only on +Linux (macOS POSIX shm segments are not path-addressable). +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import platform +import sys + +Test.Summary = ''' +An in-shm stripe directory with out-of-range header fields is rejected and rebuilt +from disk, never fast-attached. +''' +Test.ContinueOnFail = True + +# The byte-poke drives the gate by editing /dev/shm directly, which is a Linux +# facility; macOS POSIX shm is not exposed as a file. There is no Condition for +# the platform, so gate with a lambda (ports.py branches on platform the same way). +Test.SkipUnless(Condition(lambda: platform.system() == 'Linux', "shm byte-poke gates need Linux /dev/shm")) + + +class CacheShmDirInvalidTest: + """ + The per-stripe directory gates. On a fast restart the in-shm directory is used + verbatim -- the disk read and recover_data() are both skipped -- so every header + field the cache later trusts is range-checked first. A rejection must be safe and + silent to clients: the stripe falls back to the disk read, recovers, and serves + the same object. + + Sequence, per tampered field: + - poke the field in the /dev/shm stripe segment left by a clean shutdown, + - start the next ts, which must log "shm directory invalid ... falling back to + disk read" and must NOT log the fast-attach line, + - replay the cache-hit transaction, which must be served from cache. + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + POKE_SCRIPT = 'shm_poke.py' + + REPLAY_FILE = 'replay/cache-shm-dir-invalid.replay.yaml' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + # StripeHeaderFooter layout (P_CacheDir.h) at the head of the stripe segment: + # magic @0, version @4, create_time @8, write_pos @16, last_write_pos @24, + # agg_pos @32, ... sector_size @64, unused @68, freelist[0] @72. + WRITE_POS_OFFSET = 16 + FREELIST_0_OFFSET = 72 + + # Little-endian off_t 0x0000FFFFFFFFFFFF: far beyond skip + len for a 256 MiB + # stripe, so the write_pos range check rejects it. Leaving the top two bytes + # zero keeps the value positive, which exercises the upper-bound branch rather + # than the negative-offset one. + BOGUS_WRITE_POS_LE_HEX = 'ffffffffffff0000' + # Little-endian uint16 65535. The largest value a Dir next/prev field can hold, + # and past the entry count of any segment with fewer than 16384 buckets -- which + # a 256 MiB stripe is. + BOGUS_FREELIST_LE_HEX = 'ffff' + + def __init__(self): + self._setup_shared_state() + # A single verifier-server is the origin for every ts, started before ts1 + # and kept running for the whole test. + self.server = Test.MakeVerifierServerProcess('shmd-origin', self.REPLAY_FILE) + self.ts1 = self._configure_ts('shmd_ts1') + self.ts2 = self._configure_ts('shmd_ts2') + self.ts3 = self._configure_ts('shmd_ts3') + self._add_diags_log_assertions() + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + Test.Setup.Copy(os.path.join(Test.TestDirectory, self.POKE_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._shared_storage_path = os.path.join(shared_storage_dir, 'disk.img') + with open(self._shared_storage_path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'd' = dir-invalid variant. + # (This test is Linux-only, but keep the prefix short for consistency.) + self._shm_prefix = f'/cshmd-{os.getpid() % 100000}-' + # A single span with a single volume yields exactly one stripe, so its + # segment is index 0. On Linux each segment is a file under /dev/shm by the + # same name (sans the leading '/'). + self._stripe_file = '/dev/shm/' + self._shm_prefix.lstrip('/') + 's0' + + def _configure_ts(self, name): + ts = Test.MakeATSProcess(name) + # An absolute span path keeps the span independent of MakeATSProcess's + # per-instance STORAGEDIR, so every ts shares the same on-disk cache and + # therefore the same stripe geometry (hence the same shm identity). + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {self._shared_storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self.server.Variables.http_port}/') + return ts + + def _add_reject_assertions(self, ts, label): + # The shm segments themselves are still attached -- the control segment is + # untouched and clean, so this is specifically the per-stripe directory gate. + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: attaching up to \d+ stripes \(fast restart', f'{label} should attach the existing control segment') + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: attached stripe \S+ \(\d+ bytes\) for key=', f'{label} should attach the existing stripe segment') + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"shm directory invalid for '.+'; falling back to disk read", f'{label} must reject the tampered shm directory') + ts.Disk.diags_log.Content += Testers.ExcludesExpression( + r'attaching cached directory from shm for', f'{label} must not fast-attach the tampered directory') + # The rejection is per-stripe: the control segment stays valid, so none of + # the whole-segment drop reasons should appear. + ts.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: (schema|ABI) mismatch', f'{label} should reject the directory, not the control segment') + ts.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', f'{label} should see the shm marked clean') + + def _add_diags_log_assertions(self): + # ts1 cold start, clean shutdown -- a valid, clean stripe segment to tamper with. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: created stripe \S+ \(\d+ bytes\) for key=', 'ts1 should create the shm-backed stripe segment') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + self.ts1.Disk.diags_log.Content += Testers.ExcludesExpression( + r'shm directory invalid for', 'ts1 has no shm directory to reject on cold start') + + self._add_reject_assertions(self.ts2, 'ts2 (write_pos)') + self._add_reject_assertions(self.ts3, 'ts3 (freelist[0])') + + def _fill(self): + tr = Test.AddTestRun('Cold-start ts1 and cache an object') + tr.AddVerifierClientProcess( + 'shmd-fill-client', self.REPLAY_FILE, http_ports=[self.ts1.Variables.port], keys='fill', other_args='--thread-limit 1') + tr.Processes.Default.StartBefore(self.server) + tr.Processes.Default.StartBefore(self.ts1) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts1 + + def _clean_shutdown(self, ts, name): + tr = Test.AddTestRun(f'Drain and clean-shutdown {name}') + tr.Processes.Default.Env = ts.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} {name} --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + + def _poke(self, description, offset, hex_bytes): + # The previous ts is dead; the stripe segment is just a file now. + tr = Test.AddTestRun(description) + tr.Processes.Default.Command = (f'{sys.executable} ./{self.POKE_SCRIPT} {self._stripe_file} {offset} {hex_bytes}') + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + + def _verify_reject(self, ts, key, description): + tr = Test.AddTestRun(description) + tr.AddVerifierClientProcess( + f'shmd-{key}-client', self.REPLAY_FILE, http_ports=[ts.Variables.port], keys=key, other_args='--thread-limit 1') + tr.Processes.Default.StartBefore(ts) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = ts + + def _cleanup_shm(self): + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts3.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._fill() + self._clean_shutdown(self.ts1, 'shmd_ts1') + + self._poke('Tamper write_pos in the in-shm stripe header', self.WRITE_POS_OFFSET, self.BOGUS_WRITE_POS_LE_HEX) + self._verify_reject(self.ts2, 'hit_write_pos', 'Start ts2; an out-of-range write_pos is rejected and rebuilt from disk') + self._clean_shutdown(self.ts2, 'shmd_ts2') + + self._poke('Tamper freelist[0] in the in-shm stripe header', self.FREELIST_0_OFFSET, self.BOGUS_FREELIST_LE_HEX) + self._verify_reject(self.ts3, 'hit_freelist', 'Start ts3; an out-of-range freelist head is rejected and rebuilt from disk') + self._clean_shutdown(self.ts3, 'shmd_ts3') + + self._cleanup_shm() + + +CacheShmDirInvalidTest().run() diff --git a/tests/gold_tests/cache/cache_shm_fast_restart.test.py b/tests/gold_tests/cache/cache_shm_fast_restart.test.py new file mode 100644 index 00000000000..520b2b00986 --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_fast_restart.test.py @@ -0,0 +1,247 @@ +''' +Verify the cache directory survives a clean shutdown via shared memory and is +attached on the next start (fast restart). Two ATS instances share an on-disk +cache file and a POSIX shm name prefix; ts1 populates the cache and is shut +down via traffic_ctl drain + SIGTERM, then ts2 starts and serves the same URL +out of cache without re-fetching from the origin. + +Traffic is driven with Proxy Verifier: a single verifier-server acts as the +origin and the verifier-client replays cache-shm-fast-restart.replay.yaml -- +the "fill" transaction against ts1 and the "hit" transaction against ts2. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = ''' +Cache directory survives clean shutdown via POSIX shared memory. +''' +Test.ContinueOnFail = True + + +class CacheShmFastRestartTest: + """ + Cover the cache shm fast-restart scenario end-to-end. + + Sequence: + 1. ts1 cold-start: creates a fresh shm control segment and per-stripe + segments; populates the cache via the "fill" transaction (cache miss, + fetched from the verifier-server origin). + 2. ts1 is drained and SIGTERM'd. The shutdown hook flushes the directory + and marks the shm clean. + 3. ts2 starts against the same on-disk file and shm prefix: attaches the + existing control segment, attaches per-stripe segments, and reuses the + cached directory without re-reading it from disk. + 4. ts2 serves the same URL out of cache via the "hit" transaction + (X-Cache: hit-fresh). The transaction's origin response is a 502 + sentinel, so any forward to the origin would fail the run. + + Each step is verified both at the response level (proxy-verifier) and via + diags-log assertions on the cache_shm / cache_dir_init code paths. + """ + + # Helper script for sending signals to a traffic_server process by command-line + # identifier match. Reused from gold_tests/logging. + TS_PID_SCRIPT = 'ts_process_handler.py' + + # The replay file driving both the populate ("fill") and verify ("hit") + # transactions. They share a cache key and differ only by uuid. + REPLAY_FILE = 'replay/cache-shm-fast-restart.replay.yaml' + + # Stripe size for the shared cache. Must be large enough that the directory + # contains real entries; small enough that the disk.img is cheap to create. + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + def __init__(self): + self._setup_shared_state() + # A single verifier-server is the origin for both ts1 and ts2. It is + # started before ts1 and kept running across the whole test. + self.server = Test.MakeVerifierServerProcess('shm-origin', self.REPLAY_FILE) + self.ts1 = self._configure_ts('shm_ts1') + self.ts2 = self._configure_ts('shm_ts2') + self._add_diags_log_assertions() + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + # Shared storage file used by both ts1 and ts2. The absolute path makes + # storage.yaml independent of MakeATSProcess's per-instance STORAGEDIR. + # ATS opens regular-file spans with O_RDONLY first to stat them -- it + # does not auto-create the backing file -- so pre-create disk.img at the + # configured size before either ts starts. + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._shared_storage_path = os.path.join(shared_storage_dir, 'disk.img') + with open(self._shared_storage_path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # POSIX shm names: macOS PSHMNAMLEN limit is 31 chars including '/'. + # Keep the prefix short and unique per test run so concurrent autest + # runs do not collide. + self._shm_prefix = f'/cshm-{os.getpid() % 100000}-' + + def _configure_ts(self, name): + ts = Test.MakeATSProcess(name) + # Master configures cache storage via storage.yaml. An absolute span path + # keeps the span independent of MakeATSProcess's per-instance STORAGEDIR so + # ts1 and ts2 share the same on-disk cache, which yields identical stripe + # geometry (hence identical shm identity). + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {self._shared_storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + # The per-stripe 'created/attached stripe' lines are Dbg() calls; + # route debug output to diags.log (default is stderr) so the + # ContainsExpression assertions below can match them. + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self.server.Variables.http_port}/') + return ts + + def _add_diags_log_assertions(self): + # These assertions match the *stable core* of each cache-shm log line and + # deliberately stop before the trailing parenthetical qualifier. The shm + # code appends optional context to several of these messages as it evolves + # -- "attaching N stripes" became "attaching up to N stripes", + # "(fast restart)" became "(fast restart, recovery skipped)" on the stripe + # path and "(fast restart, partial -- storage changed)" on the control + # path. Anchoring on the invariant prefix (not the closing paren) keeps the + # test from breaking every time such a qualifier is added. Likewise, the + # excludes name only log strings that actually exist in the source: an + # exclude on a non-existent string can never fire and gives false comfort. + + # ts1 (cold start): creates fresh shm, marks it clean on shutdown, and must + # NOT report any "drop" reason since there is nothing to drop. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: created stripe \S+ \(\d+ bytes\) for key=', 'ts1 should create at least one shm-backed stripe segment') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + self.ts1.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: (schema|ABI) mismatch', 'ts1 should not detect any shm mismatch on cold start') + self.ts1.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', 'ts1 should not see a dirty shm on cold start') + self.ts1.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: stripe \S+ size mismatch', 'ts1 should not see a stripe size mismatch on cold start') + + # ts2 (warm start): attaches the existing control segment, fast-attaches the + # per-stripe segment, reuses the cached directory, and must NOT fall back to + # the disk-rebuild path. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: attaching up to \d+ stripes \(fast restart', 'ts2 should attach the existing shm (fast restart)') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: attached stripe \S+ \(\d+ bytes\) for key=', 'ts2 should attach at least one shm-backed stripe segment') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r"attaching cached directory from shm for '.+' \(fast restart", 'ts2 should reuse the per-stripe directory from shm') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: creating fresh control segment', 'ts2 should not create a fresh control segment') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: (schema|ABI) mismatch', 'ts2 should not detect any shm mismatch on warm start') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', 'ts2 should see the shm marked clean') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'shm directory invalid for', 'ts2 should not fall back from shm to disk read') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: stripe \S+ size mismatch', 'ts2 should fast-attach without a stripe size-mismatch recreate') + + def _start_ts1(self): + # Cold start ts1 against the verifier-server origin and replay the + # "fill" transaction: a cache miss that ATS fetches and stores. + tr = Test.AddTestRun('Start ts1, then cache contents (fill)') + tr.AddVerifierClientProcess( + 'shm-fill-client', self.REPLAY_FILE, http_ports=[self.ts1.Variables.port], keys='fill', other_args='--thread-limit 1') + tr.Processes.Default.StartBefore(self.server) + tr.Processes.Default.StartBefore(self.ts1) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts1 + + def _clean_shutdown_ts1(self): + # Drain + SIGTERM ts1. SIGTERM goes through AutoStopCont which invokes + # TS_LIFECYCLE_SHUTDOWN_HOOK -> sync_cache_dir_on_shutdown -> + # CacheShm::mark_clean_shutdown. + tr = Test.AddTestRun('Drain and clean-shutdown ts1') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shm_ts1 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + + def _start_ts2(self): + # ts2 attaches the CacheDir from the shm created by ts1. Replay the + # "hit" transaction: ATS must serve it from cache (X-Cache: hit-fresh) + # without contacting the origin -- the replay's 502 sentinel response + # would otherwise surface as a proxy-response mismatch. + tr = Test.AddTestRun('Start ts2; verify shm fast-attach and cache HIT') + tr.AddVerifierClientProcess( + 'shm-hit-client', self.REPLAY_FILE, http_ports=[self.ts2.Variables.port], keys='hit', other_args='--thread-limit 1') + tr.Processes.Default.StartBefore(self.ts2) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown_ts2(self): + # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a + # segment a live traffic_server still owns, so the segments must be + # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + tr = Test.AddTestRun('Drain and clean-shutdown ts2') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shm_ts2 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + + def _cleanup_shm(self): + # A clean shutdown deliberately keeps the control + live stripe segments + # for the next fast restart, so they outlive the test. Unlink them by + # prefix to avoid leaking POSIX shm across repeated local runs (macOS has + # no /dev/shm to clear out of band). + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._start_ts1() + self._clean_shutdown_ts1() + self._start_ts2() + self._clean_shutdown_ts2() + self._cleanup_shm() + + +CacheShmFastRestartTest().run() diff --git a/tests/gold_tests/cache/cache_shm_purge_on_disable.test.py b/tests/gold_tests/cache/cache_shm_purge_on_disable.test.py new file mode 100644 index 00000000000..51fd57cdedf --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_purge_on_disable.test.py @@ -0,0 +1,232 @@ +''' +Purge-stale-on-start: when shm is disabled but a prior run left segments behind, +proxy.config.cache.shm.purge_stale_on_start=1 removes them at startup. + +This guards two hazards of running with the feature disabled after it had been +enabled (see records.yaml docs): (a) the leftover segments keep consuming tmpfs +the disabled instance never reads, and (b) a later re-enabled run would otherwise +fast-attach a directory that went stale while ATS ran disabled (writing only to +disk). + +Three scenarios, each on its own shm prefix + on-disk storage so they do not +interact: + + - PURGE (positive): a seed instance runs shm-enabled and clean-shuts-down, + leaving a clean control + stripe segment. A second instance runs disabled + with purge_stale_on_start=1 and must remove them. Confirmed three ways: the + seed's "clean" segment exists before (traffic_ctl cache shm status, exit 0), + the disabled instance logs the purge Note, and the segment is gone after + (status exits 2, "not found"). + + - KEEP (negative): same seed, but the disabled instance has + purge_stale_on_start=0. It must NOT log the purge and the segment must remain. + + - NOOP (no leftover): a disabled instance with purge_stale_on_start=1 against a + never-used prefix must do nothing quietly -- no purge Note, no "cannot open" + warning. + +The segments are inspected with traffic_ctl (POSIX shm is not path-addressable on +macOS, so /dev/shm cannot be listed directly). +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import sys + +Test.Summary = ''' +shm.purge_stale_on_start removes leftover shm segments at startup when shm is +disabled, only when set, and only when a control segment exists. +''' +Test.ContinueOnFail = True + + +class CacheShmPurgeOnDisableTest: + + TS_PID_SCRIPT = 'ts_process_handler.py' + DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB; matches the other shm gold tests. + + # CtrlCommand sets this exit code when a shm control segment is absent/invalid + # (src/traffic_ctl/TrafficCtlStatus.h). + CTRL_EX_ERROR = 2 + + PURGE_NOTE = r"cache shm: purged stale segments while disabled \(removed [1-9]" + + def __init__(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + pid = os.getpid() % 100000 + # Each control name is "control"; keep well under macOS PSHMNAMLEN (31). + self._prefix_purge = f'/cshmp-{pid}-' # positive: must be purged + self._prefix_keep = f'/cshmk-{pid}-' # negative: must remain + self._prefix_noop = f'/cshmz-{pid}-' # no leftover: nothing to do + + # Seed (shm enabled) instances that create the leftover segments. + self.seed_purge = self._make_ts('cshm_seed_p', self._prefix_purge, 'disk_p.img', enabled=True, purge=False) + self.seed_keep = self._make_ts('cshm_seed_k', self._prefix_keep, 'disk_k.img', enabled=True, purge=False) + + # Disabled instances under test. + self.run_purge = self._make_ts('cshm_run_p', self._prefix_purge, 'disk_p.img', enabled=False, purge=True) + self.run_keep = self._make_ts('cshm_run_k', self._prefix_keep, 'disk_k.img', enabled=False, purge=False) + self.run_noop = self._make_ts('cshm_run_z', self._prefix_noop, 'disk_z.img', enabled=False, purge=True) + + self._add_diags_assertions() + + def _make_ts(self, name, prefix, disk_name, enabled, purge): + disk_path = self._ensure_disk(disk_name) + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {disk_path}', + f' size: {self.DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.remap_config.AddLine('map / http://127.0.0.1:8080/') # never exercised; keeps remap.config non-empty + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1 if enabled else 0, + 'proxy.config.cache.shm.name_prefix': prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.cache.shm.purge_stale_on_start': 1 if purge else 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + return ts + + def _ensure_disk(self, disk_name): + storage_dir = os.path.join(Test.RunDirectory, 'storage') + os.makedirs(storage_dir, exist_ok=True) + path = os.path.join(storage_dir, disk_name) + if not os.path.exists(path): + with open(path, 'ab') as f: + f.truncate(self.DISK_SIZE_BYTES) + return path + + def _add_diags_assertions(self): + # Seeds create a fresh control segment and mark it clean on the way out -- + # that is the "fast-attachable but now stale" state the purge must clean up. + for seed in (self.seed_purge, self.seed_keep): + seed.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'seed should create a fresh shm control segment') + seed.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'seed should mark the shm clean before exit') + + # Positive: the disabled+purge instance logs the purge of at least one segment. + self.run_purge.Disk.diags_log.Content += Testers.ContainsExpression( + self.PURGE_NOTE, 'disabled instance with purge_stale_on_start=1 should purge the leftover segments') + + # Negative: purge_stale_on_start=0 must never purge. + self.run_keep.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: purged stale segments', 'purge_stale_on_start=0 must not purge') + + # No-op: nothing exists for this prefix, so neither a purge nor an error. + self.run_noop.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: purged stale segments', 'no leftover means nothing is purged') + self.run_noop.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: cannot open control segment', 'a missing control segment is a quiet no-op, not a warning') + + def _shm_status(self, description, ts, prefix, expect_present): + """Run `traffic_ctl cache shm status` and assert the control segment is (not) there.""" + control_name = prefix + 'control' + tr = Test.AddTestRun(description) + tr.Processes.Default.Env = ts.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm status --prefix {prefix}' + if expect_present: + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + r'Control segment:\s+' + re.escape(control_name), 'control segment should be present') + else: + tr.Processes.Default.ReturnCode = self.CTRL_EX_ERROR + tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + r"control segment '" + re.escape(control_name) + r"' not found", 'control segment should be gone') + return tr + + def _start_seed(self, description, seed, prefix): + # Starting the seed (shm enabled) creates the control + stripe segments; the + # status probe also confirms they exist while the seed is the live owner. + tr = self._shm_status(description, seed, prefix, expect_present=True) + tr.Processes.Default.StartBefore(seed) + tr.StillRunningAfter = seed + return tr + + def _clean_shutdown(self, description, seed, name): + tr = Test.AddTestRun(description) + tr.Processes.Default.Env = seed.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} {name} --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _start_disabled(self, description, ts, prefix, expect_present): + # Start the disabled instance under test; its purge (or no-op) runs during + # cache init, so by the time it is ready the status below reflects the result. + tr = self._shm_status(description, ts, prefix, expect_present=expect_present) + tr.Processes.Default.StartBefore(ts) + tr.StillRunningAfter = ts + return tr + + def _cleanup(self): + tr = Test.AddTestRun('Unlink any remaining test shm segments') + tr.Processes.Default.Env = self.run_keep.Env + tr.Processes.Default.Command = ( + f'traffic_ctl cache shm clear --prefix {self._prefix_purge} ; ' + f'traffic_ctl cache shm clear --prefix {self._prefix_keep} ; ' + f'traffic_ctl cache shm clear --prefix {self._prefix_noop}') + tr.Processes.Default.ReturnCode = 0 + + def run(self): + # PURGE (positive) + self._start_seed('PURGE: start shm-enabled seed; control segment is created', self.seed_purge, self._prefix_purge) + self._clean_shutdown('PURGE: clean-shutdown seed (leaves a clean segment)', self.seed_purge, 'cshm_seed_p') + # Probe with a Env whose bin/ autest has already populated (seed_purge was + # started above); run_purge has not started yet, so its bin/ does not exist. + self._shm_status( + 'PURGE: precondition -- clean leftover segment is present', self.seed_purge, self._prefix_purge, + expect_present=True).Processes.Default.Streams.stdout += Testers.ContainsExpression( + r'clean_shutdown:\s+1 \(clean\)', 'leftover segment should be marked clean (the stale-but-attachable case)') + self._start_disabled( + 'PURGE: start disabled+purge=1; leftover segments are removed', + self.run_purge, + self._prefix_purge, + expect_present=False) + + # KEEP (negative) + self._start_seed('KEEP: start shm-enabled seed; control segment is created', self.seed_keep, self._prefix_keep) + self._clean_shutdown('KEEP: clean-shutdown seed (leaves a clean segment)', self.seed_keep, 'cshm_seed_k') + self._start_disabled( + 'KEEP: start disabled+purge=0; leftover segments remain', self.run_keep, self._prefix_keep, expect_present=True) + + # NOOP (no leftover) + tr = Test.AddTestRun('NOOP: start disabled+purge=1 against an unused prefix; nothing to do') + tr.Processes.Default.Env = self.run_noop.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm status --prefix {self._prefix_noop}' + tr.Processes.Default.ReturnCode = self.CTRL_EX_ERROR # never existed + tr.Processes.Default.StartBefore(self.run_noop) + tr.StillRunningAfter = self.run_noop + + self._cleanup() + + +CacheShmPurgeOnDisableTest().run() diff --git a/tests/gold_tests/cache/cache_shm_schema_mismatch.test.py b/tests/gold_tests/cache/cache_shm_schema_mismatch.test.py new file mode 100644 index 00000000000..fac47ebde57 --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_schema_mismatch.test.py @@ -0,0 +1,207 @@ +''' +Verify the shm schema-version trust gate: a control segment whose schema_version +does not match the running build is dropped, never fast-attached. ts1 cold-starts, +caches an object, and clean-shuts-down (marking the segment clean). The segment +file under /dev/shm is then tampered -- schema_version is overwritten with a +bogus value -- before ts2 starts against the same shm prefix. ts2 must detect the +mismatch, drop the segment, recreate it fresh, and rebuild the directory from disk. + +Linux-only: it pokes raw bytes in the /dev/shm segment file, which exists only on +Linux (macOS POSIX shm segments are not path-addressable). +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import platform +import sys +import uuid + +Test.Summary = ''' +A control segment with a mismatched schema_version is dropped and rebuilt from +disk, never fast-attached. +''' +Test.ContinueOnFail = True + +# The byte-poke drives the gate by editing /dev/shm directly, which is a Linux +# facility; macOS POSIX shm is not exposed as a file. There is no Condition for +# the platform, so gate with a lambda (ports.py branches on platform the same way). +Test.SkipUnless(Condition(lambda: platform.system() == 'Linux', "shm byte-poke gates need Linux /dev/shm")) + + +class CacheShmSchemaMismatchTest: + """ + The schema-version gate. The control header records the build's + CACHE_SHM_SCHEMA_VERSION; on attach, a segment whose recorded version differs + is dropped ("schema mismatch ( vs ), dropping") rather than trusted + -- the on-disk struct layout it describes may no longer match this build. The + ABI-hash gate (abi_hash @16) works identically; this test exercises the + schema field (@8) as the representative case. + + Sequence: ts1 creates a clean segment, then schema_version is poked to a bogus + value, then ts2 starts and must: + - log the schema mismatch and drop, + - recreate a fresh control segment, + - NOT fast-attach, + - and still serve a request (200). + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + POKE_SCRIPT = 'shm_poke.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + # CacheShmControl layout (CacheShmLayout.h): magic[8] @0, schema_version @8. + SCHEMA_VERSION_OFFSET = 8 + # Little-endian uint32 = 9; the build's CACHE_SHM_SCHEMA_VERSION is small, so + # any value it never uses works. 9 is comfortably out of range. + BOGUS_SCHEMA_LE_HEX = '09000000' + + def __init__(self): + self._setup_shared_state() + self.ts1 = self._configure_ts('shmx_ts1') + self.ts2 = self._configure_ts('shmx_ts2') + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + Test.Setup.Copy(os.path.join(Test.TestDirectory, self.POKE_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._shared_storage_path = os.path.join(shared_storage_dir, 'disk.img') + with open(self._shared_storage_path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'x' = schema-mismatch variant. + # (This test is Linux-only, but keep the prefix short for consistency.) + self._shm_prefix = f'/cshmx-{os.getpid() % 100000}-' + # The control segment is name_prefix + "control"; on Linux it is a file + # under /dev/shm by the same name (sans the leading '/'). + self._control_file = '/dev/shm/' + self._shm_prefix.lstrip('/') + 'control' + + def _configure_ts(self, name): + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {self._shared_storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 cold start, clean shutdown -- a valid, clean segment to tamper with. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + + # ts2 start against the poked segment: detect, drop, recreate, rebuild. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: schema mismatch \(\d+ vs \d+\), dropping', 'ts2 must detect the schema mismatch and drop the segment') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts2 must recreate the control segment after the drop') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'\(fast restart, recovery skipped\)', 'ts2 must rebuild from disk, never fast-attach the mismatched segment') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', + 'the drop must be due to the schema mismatch, not an unclean shutdown') + + def _populate_cache(self): + tr = Test.AddTestRun('Cold-start ts1 and cache an object') + tr.Processes.Default.StartBefore(self.ts1) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts1.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', 'ts1 first GET should return 200') + tr.StillRunningAfter = self.ts1 + + def _clean_shutdown_ts1(self): + tr = Test.AddTestRun('Drain and clean-shutdown ts1') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shmx_ts1 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _poke_schema_version(self): + # ts1 is dead; the segment is just a file now. Overwrite schema_version. + tr = Test.AddTestRun('Tamper schema_version in the shm control segment') + tr.Processes.Default.Command = ( + f'{sys.executable} ./{self.POKE_SCRIPT} {self._control_file} ' + f'{self.SCHEMA_VERSION_OFFSET} {self.BOGUS_SCHEMA_LE_HEX}') + tr.Processes.Default.ReturnCode = 0 + + def _verify_mismatch_drop(self): + tr = Test.AddTestRun('Start ts2; verify the schema mismatch is dropped and rebuilt from disk') + tr.Processes.Default.StartBefore(self.ts2) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts2.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + '200', 'ts2 should serve correctly after dropping the mismatched segment') + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown_ts2(self): + tr = Test.AddTestRun('Drain and clean-shutdown ts2') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shmx_ts2 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._populate_cache() + self._clean_shutdown_ts1() + self._poke_schema_version() + self._verify_mismatch_drop() + self._clean_shutdown_ts2() + self._cleanup_shm() + + +CacheShmSchemaMismatchTest().run() diff --git a/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py b/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py new file mode 100644 index 00000000000..ad4262f8052 --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py @@ -0,0 +1,210 @@ +''' +Verify that a changed storage layout never fast-attaches a stale directory. +A storage.yaml change no longer drops the whole shm control segment; instead +each stripe is matched to its prior segment by its own identity. ts1 caches an +object against one storage file and clean-shuts-down (marking the shm clean); +ts2 starts against a *different* storage file but the *same* shm name prefix. +ts2 finds ts1's control segment, keeps it (partial attach), but because its +stripe identity no longer matches any recorded entry it creates a fresh stripe +segment and reclaims ts1's orphaned one -- it must never fast-attach a segment +that describes a different on-disk layout. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import uuid + +Test.Summary = ''' +A changed storage layout never fast-attaches a stale directory: the control +segment is kept (partial attach), the relocated stripe creates a fresh segment, +and the orphaned prior segment is reclaimed. +''' +Test.ContinueOnFail = True + + +class CacheShmStorageMismatchTest: + """ + The storage signature is a fingerprint of every span's path and geometry, + stored in the shm control header. It is no longer a hard gate: a storage + change keeps the control segment and lets each stripe attach by its own + identity (its hash_text, which includes the disk path). This test points + ts1 and ts2 at different storage files (a repath) while sharing one shm + prefix, and asserts ts2: + - keeps the existing control segment (does NOT recreate it), + - enters partial-attach mode because the storage signature changed, + - never fast-attaches any stripe segment (its identity differs, so the + stale directory built for storage A is never reused), + - creates a fresh stripe segment for its own (storage B) layout, + - reclaims ts1's now-orphaned stripe segment, + - and still serves a request (200). + Because the storage change does not gate the clean-shutdown check, ts2 must + NOT report the prior run as unclean: the only reason for the recreate is the + storage change. + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + def __init__(self): + self._setup_shared_state() + # ts1 and ts2 share the shm prefix but use different storage files. + self.ts1 = self._configure_ts('shms_ts1', self._storage_path_a) + self.ts2 = self._configure_ts('shms_ts2', self._storage_path_b) + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + # Two distinct storage files -> distinct span paths -> distinct + # storage signatures, which is exactly the "repath" case under test. + self._storage_path_a = os.path.join(shared_storage_dir, 'disk_a.img') + self._storage_path_b = os.path.join(shared_storage_dir, 'disk_b.img') + for path in (self._storage_path_a, self._storage_path_b): + with open(path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 's' = storage-mismatch variant. + self._shm_prefix = f'/cshms-{os.getpid() % 100000}-' + + def _configure_ts(self, name, storage_path): + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 cold start against storage A, clean shutdown. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'created stripe \S+ \(\d+ bytes\) for key=', 'ts1 should create at least one shm-backed stripe segment') + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: marking clean shutdown', 'ts1 should mark the shm clean before exit') + + # ts2 start against storage B: the storage signature differs, so the + # control segment is kept (partial attach) but the relocated stripe + # creates a fresh segment rather than fast-attaching the stale one. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'attaching up to \d+ stripes \(fast restart, partial -- storage changed\)', + 'ts2 must enter partial-attach mode after the storage change') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'created stripe \S+ \(\d+ bytes\) for key=', 'ts2 must create a fresh stripe segment for its own layout') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: reclaiming orphaned stripe segment', "ts2 must reclaim ts1's orphaned stripe segment") + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'reclaimed \d+ orphaned stripe segment\(s\) after attach', 'ts2 must report the reclaim summary') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'attached stripe \S+ \(\d+ bytes\) for key=', + 'ts2 must never fast-attach a stripe segment built for a different layout') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: creating fresh control segment', 'ts2 must keep the control segment across the storage change') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: (schema|ABI) mismatch', 'the recreate must be due to the storage change, not schema/ABI') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: previous run did not shutdown cleanly', + 'the recreate must be due to the storage change, not an unclean shutdown') + + def _populate_cache(self): + tr = Test.AddTestRun('Populate cache via ts1 (storage A)') + tr.Processes.Default.StartBefore(self.ts1) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts1.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', 'ts1 first GET should return 200') + tr.StillRunningAfter = self.ts1 + + def _clean_shutdown_ts1(self): + tr = Test.AddTestRun('Drain and clean-shutdown ts1') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shms_ts1 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _verify_partial_attach_and_reclaim(self): + tr = Test.AddTestRun('Start ts2 (storage B); verify partial attach: fresh stripe + orphan reclaim') + tr.Processes.Default.StartBefore(self.ts2) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts2.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + '200', 'ts2 should serve correctly after the partial attach') + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown_ts2(self): + # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a + # segment a live traffic_server still owns, so the segments must be + # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + tr = Test.AddTestRun('Drain and clean-shutdown ts2') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shms_ts2 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + # A clean shutdown deliberately keeps the control + live stripe segments + # for the next fast restart, so they outlive the test. Unlink them by + # prefix to avoid leaking POSIX shm across repeated local runs (macOS has + # no /dev/shm to clear out of band). + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._populate_cache() + self._clean_shutdown_ts1() + self._verify_partial_attach_and_reclaim() + self._clean_shutdown_ts2() + self._cleanup_shm() + + +CacheShmStorageMismatchTest().run() diff --git a/tests/gold_tests/cache/cache_shm_unclean_shutdown.test.py b/tests/gold_tests/cache/cache_shm_unclean_shutdown.test.py new file mode 100644 index 00000000000..4711636c70b --- /dev/null +++ b/tests/gold_tests/cache/cache_shm_unclean_shutdown.test.py @@ -0,0 +1,185 @@ +''' +Verify the shm fast-restart path refuses to trust a directory left by a crash. +A clean shutdown marks the control segment clean; a crash (SIGKILL) does not, +so clean_shutdown stays 0. ts1 cold-starts, caches an object, and is *killed* +(no drain, no SIGTERM) so the shutdown hook never runs. ts2 starts against the +same on-disk cache and shm prefix: it must find the dirty segment, drop the +whole thing, rebuild the directory from disk, and never take the fast-attach +"recovery skipped" path. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import uuid + +Test.Summary = ''' +An unclean shutdown (SIGKILL) leaves the shm control segment dirty, so the next +start drops it and rebuilds the directory from disk instead of fast-attaching. +''' +Test.ContinueOnFail = True + + +class CacheShmUncleanShutdownTest: + """ + The crash-safety gate. clean_shutdown is set to 1 only by the shutdown hook + (CacheShm::mark_clean_shutdown); a SIGKILL bypasses it, leaving the segment + with clean_shutdown == 0. On the next start the control segment is found but + rejected -- a crash may have left dir entries pointing at content that never + reached disk, so no stripe can safely skip recovery. ts2 must: + - log "previous run did not shutdown cleanly, dropping", + - recreate a fresh control segment, + - NOT take the stripe fast-attach "recovery skipped" path, + - and still serve a request (200). + + This gate is cross-platform: clean_shutdown lives in the control segment, so + it does not depend on the Linux-only flock path. + """ + + TS_PID_SCRIPT = 'ts_process_handler.py' + + SHARED_DISK_SIZE_BYTES = 256 * 1024 * 1024 # 256 MiB + + def __init__(self): + self._setup_shared_state() + # ts1 and ts2 share the same on-disk cache file and shm prefix so ts2 + # would fast-attach ts1's directory -- were it not left dirty by the kill. + self.ts1 = self._configure_ts('shmu_ts1') + # ts1 is SIGKILLed mid-test, so it exits on signal 9 (returncode -9, or + # 137 where the runner reports 128+signal). Declare that expected exit so + # the managed-process check does not flag the deliberate kill. ts1 still + # starts normally, so leave Ready at its default (port-open) condition. + self.ts1.ReturnCode = Any(-9, 137) + self.ts2 = self._configure_ts('shmu_ts2') + self._add_diags_log_assertions() + self._url_path = f'/cache/40/{uuid.uuid4()}' + + def _setup_shared_state(self): + Test.Setup.Copy(os.path.join(Test.TestDirectory, '..', 'logging', self.TS_PID_SCRIPT)) + + shared_storage_dir = os.path.join(Test.RunDirectory, 'shared-storage') + os.makedirs(shared_storage_dir, exist_ok=True) + self._shared_storage_path = os.path.join(shared_storage_dir, 'disk.img') + with open(self._shared_storage_path, 'ab') as f: + f.truncate(self.SHARED_DISK_SIZE_BYTES) + + # macOS PSHMNAMLEN is 31 chars incl. '/'; 'u' = unclean-shutdown variant. + self._shm_prefix = f'/cshmu-{os.getpid() % 100000}-' + + def _configure_ts(self, name): + ts = Test.MakeATSProcess(name) + ts.Disk.storage_yaml.AddLines( + [ + 'cache:', + ' spans:', + ' - name: disk.0', + f' path: {self._shared_storage_path}', + f' size: {self.SHARED_DISK_SIZE_BYTES}', + ' volumes:', + ' - id: 1', + ' scheme: http', + ' size: 100%', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.cache.shm.enabled': 1, + 'proxy.config.cache.shm.name_prefix': self._shm_prefix, + 'proxy.config.cache.shm.use_hugepages': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_shm', + 'proxy.config.diags.output.diag': 'L', + 'proxy.config.http.wait_for_cache': 1, + }) + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache,via') + ts.Disk.remap_config.AddLine('map / http://127.0.0.1/ @plugin=generator.so') + return ts + + def _add_diags_log_assertions(self): + # ts1 cold start: creates a fresh segment but is killed before it can mark + # the shutdown clean. + self.ts1.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts1 should create a fresh shm control segment on first start') + self.ts1.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: marking clean shutdown', 'ts1 is SIGKILLed, so it must never mark the shm clean') + + # ts2 start: finds the dirty segment, drops it, recreates, and rebuilds + # from disk -- it must NOT fast-attach. + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: previous run did not shutdown cleanly, dropping', 'ts2 must reject the dirty segment left by the crash') + self.ts2.Disk.diags_log.Content += Testers.ContainsExpression( + r'cache shm: creating fresh control segment', 'ts2 must recreate the control segment after dropping the dirty one') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'\(fast restart, recovery skipped\)', 'ts2 must rebuild from disk, never take the fast-attach path') + self.ts2.Disk.diags_log.Content += Testers.ExcludesExpression( + r'cache shm: attaching up to \d+ stripes \(fast restart', 'ts2 must not attach the dirty control segment') + + def _populate_cache(self): + tr = Test.AddTestRun('Cold-start ts1 and cache an object') + tr.Processes.Default.StartBefore(self.ts1) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts1.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression('200', 'ts1 first GET should return 200') + tr.StillRunningAfter = self.ts1 + + def _kill_ts1(self): + # SIGKILL -- no drain, no SIGTERM -- so the shutdown hook never runs and + # the control segment is left with clean_shutdown == 0. + tr = Test.AddTestRun('SIGKILL ts1 (unclean shutdown)') + tr.Processes.Default.Env = self.ts1.Env + tr.Processes.Default.Command = (f'{sys.executable} ./{self.TS_PID_SCRIPT} shmu_ts1 --signal KILL && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _verify_dirty_drop(self): + tr = Test.AddTestRun('Start ts2; verify the dirty segment is dropped and rebuilt from disk') + tr.Processes.Default.StartBefore(self.ts2) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}\\n" ' + f'-H "x-debug: x-cache,via" ' + f'http://127.0.0.1:{self.ts2.Variables.port}{self._url_path}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + '200', 'ts2 should serve correctly after dropping the dirty segment') + tr.StillRunningAfter = self.ts2 + + def _clean_shutdown_ts2(self): + tr = Test.AddTestRun('Drain and clean-shutdown ts2') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = ( + f'traffic_ctl server drain && sleep 1 && ' + f'{sys.executable} ./{self.TS_PID_SCRIPT} shmu_ts2 --signal TERM && sleep 3') + tr.Processes.Default.ReturnCode = 0 + + def _cleanup_shm(self): + tr = Test.AddTestRun('Unlink the test shm segments') + tr.Processes.Default.Env = self.ts2.Env + tr.Processes.Default.Command = f'traffic_ctl cache shm clear --prefix {self._shm_prefix}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stderr = Testers.ExcludesExpression( + 'Invalid argument', 'clear must skip tombstoned slots, not fail on them') + + def run(self): + self._populate_cache() + self._kill_ts1() + self._verify_dirty_drop() + self._clean_shutdown_ts2() + self._cleanup_shm() + + +CacheShmUncleanShutdownTest().run() diff --git a/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold b/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold new file mode 100644 index 00000000000..5a981414f4f --- /dev/null +++ b/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold @@ -0,0 +1,13 @@ +Control segment: `` + segment size: `` + magic: `` [valid] + schema_version: `` [valid] + abi_hash: 0x`` + storage_sig: 0x`` + clean_shutdown: 1 (clean) + owner_pid: 0 (none -- not currently attached) + stripe_count: 2 + +Stripes: + [0] `` present + [1] `` present diff --git a/tests/gold_tests/cache/replay/cache-shm-dir-invalid.replay.yaml b/tests/gold_tests/cache/replay/cache-shm-dir-invalid.replay.yaml new file mode 100644 index 00000000000..f83176c5c67 --- /dev/null +++ b/tests/gold_tests/cache/replay/cache-shm-dir-invalid.replay.yaml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Traffic for the shm stripe-directory rejection test. All three transactions +# share a cache key (same method + host + url) and differ only in the uuid, so +# the verifier-client can drive them one at a time via --keys: +# +# * key "fill": replayed against ts1 (cold cache). The request misses, ATS +# forwards it to the origin (verifier-server) and caches the 200 response. +# +# * keys "hit_write_pos" / "hit_freelist": replayed against ts2 / ts3 after the +# in-shm stripe header has been tampered with. Each instance must reject the +# shm directory, rebuild it from disk, and still serve the object out of cache +# (X-Cache: hit-fresh). The 502 server-response is a sentinel: it is only ever +# returned if ATS wrongly forwards the request to the origin, in which case +# the proxy-response check fails the run. +# + +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /cache-shm-dir-invalid/object + headers: + fields: + - [ Host, example.com ] + - [ uuid, fill ] + - [ X-Debug, "x-cache,via" ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 16 ] + - [ Cache-Control, "max-age=300,public" ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cache, { value: miss, as: equal } ] + + # Tamper write_pos in the in-shm stripe header, then restart ATS. + + - client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /cache-shm-dir-invalid/object + headers: + fields: + - [ Host, example.com ] + - [ uuid, hit_write_pos ] + - [ X-Debug, "x-cache,via" ] + + server-response: + status: 502 + reason: "Bad Gateway" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Tamper freelist[0] in the in-shm stripe header, then restart ATS. + + - client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /cache-shm-dir-invalid/object + headers: + fields: + - [ Host, example.com ] + - [ uuid, hit_freelist ] + - [ X-Debug, "x-cache,via" ] + + server-response: + status: 502 + reason: "Bad Gateway" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cache, { value: hit-fresh, as: equal } ] diff --git a/tests/gold_tests/cache/replay/cache-shm-fast-restart.replay.yaml b/tests/gold_tests/cache/replay/cache-shm-fast-restart.replay.yaml new file mode 100644 index 00000000000..a18a9951643 --- /dev/null +++ b/tests/gold_tests/cache/replay/cache-shm-fast-restart.replay.yaml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Traffic for the cache shm fast-restart test. The two transactions share a +# cache key (same method + host + url); they differ only in the uuid so the +# verifier-client can drive them one at a time via --keys: +# +# * key "fill": replayed against ts1 (cold cache). The request misses, ATS +# forwards it to the origin (verifier-server) and caches the 200 response. +# +# * key "hit": replayed against ts2 after a clean shutdown + shm fast restart. +# ATS must serve it from the shm-attached directory WITHOUT contacting the +# origin. The 502 server-response is a sentinel: it is only ever returned if +# ATS wrongly forwards the request, in which case the proxy-response check +# (expecting the cached 200 / X-Cache: hit-fresh) fails. +# + +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /cache-shm-fast-restart/object + headers: + fields: + - [ Host, example.com ] + - [ uuid, fill ] + - [ X-Debug, "x-cache,via" ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 16 ] + - [ Cache-Control, "max-age=300,public" ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cache, { value: miss, as: equal } ] + + # Restart ATS + + - client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /cache-shm-fast-restart/object + headers: + fields: + - [ Host, example.com ] + - [ uuid, hit ] + - [ X-Debug, "x-cache,via" ] + + server-response: + status: 502 + reason: "Bad Gateway" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cache, { value: hit-fresh, as: equal } ] diff --git a/tests/gold_tests/cache/shm_poke.py b/tests/gold_tests/cache/shm_poke.py new file mode 100644 index 00000000000..a4ed38c0485 --- /dev/null +++ b/tests/gold_tests/cache/shm_poke.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Flip raw bytes in a cache shm segment file, for shm trust-gate autests. + +On Linux the POSIX shm segments are plain files under /dev/shm, so a segment +left behind by a clean shutdown can be tampered with between runs to drive the +control-segment trust gates (schema/ABI mismatch, an unterminated shm_name, +etc.). This is Linux-only: macOS POSIX shm segments are not path-addressable. + +Usage: + shm_poke.py + +Example (set schema_version @8 to 9, little-endian uint32): + shm_poke.py /dev/shm/cshmx-12345-control 8 09000000 +""" + +import sys + + +def main() -> int: + if len(sys.argv) != 4: + sys.stderr.write(f'usage: {sys.argv[0]} \n') + return 2 + path = sys.argv[1] + offset = int(sys.argv[2], 0) + data = bytes.fromhex(sys.argv[3]) + with open(path, 'r+b') as f: + f.seek(offset) + f.write(data) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/logging/ts_process_handler.py b/tests/gold_tests/logging/ts_process_handler.py index 40640e3922e..d0d25b7487a 100644 --- a/tests/gold_tests/logging/ts_process_handler.py +++ b/tests/gold_tests/logging/ts_process_handler.py @@ -36,8 +36,8 @@ def __init__(self, message): def get_ts_process_pid(ts_identifier): processes = [] for proc in psutil.process_iter(['cmdline']): - cmdline = proc.info.get('cmdline', []) - if not cmdline: + cmdline = proc.info.get('cmdline') + if not cmdline: # None (unreadable) or empty continue commandline = ' '.join(cmdline) if '/traffic_server' in commandline and ts_identifier in commandline: From 08453f382fbf616c4f6647da73644a1464d46e5a Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 4 Aug 2026 07:40:19 +0900 Subject: [PATCH 2/7] traffic_ctl: run cache shm subcommands through Command_Execute The status/clear leaves passed their own `[&]() { command->execute(); }` lambda, where every other leaf in the file passes Command_Execute. Reuse it so the null guard applies and the wiring is uniform. No behavior change: `command` is assigned before args.invoke() and a null one throws, so the guard cannot fire today. --- src/traffic_ctl/traffic_ctl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index d5ba581419b..8bba5ae31e1 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -328,10 +328,10 @@ main([[maybe_unused]] int argc, const char **argv) // one. CacheShmCommand supplies the runtime default and keys its "did you set // name_prefix?" hint off the option being absent. shm_command.add_option("--prefix", "-p", "shm name prefix word, framed as /- (default 'ats')", "", 1, ""); - shm_command.add_command("status", "Show the cache shared-memory control segment and stripe table", [&]() { command->execute(); }) + shm_command.add_command("status", "Show the cache shared-memory control segment and stripe table", Command_Execute) .add_example_usage("traffic_ctl cache shm status") .add_example_usage("traffic_ctl cache shm status --prefix ats-t"); - shm_command.add_command("clear", "Unlink the cache shared-memory control and stripe segments", [&]() { command->execute(); }) + shm_command.add_command("clear", "Unlink the cache shared-memory control and stripe segments", Command_Execute) .add_example_usage("traffic_ctl cache shm clear") .add_example_usage("traffic_ctl cache shm clear --prefix ats-t"); From 8415a110a830594a1b2b9d5f71f1e493ad02ce3e Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 4 Aug 2026 07:40:39 +0900 Subject: [PATCH 3/7] cache: fix shm shutdown test where flock is honored The untrusted-entry test simulated a restart by calling initialize() again in one process, which the concurrent-attach guard correctly refuses: the first start still holds LOCK_EX on the control fd, and the second open of the same object conflicts with it. The test only passed where flock is not honored for POSIX shm (macOS, FreeBSD) and failed on Fedora and Debian; the other Linux builds compile the feature out, so the target is not built there at all. Add CacheShm::release_for_test() to drop the process-wide state, standing in for the process exit that releases the flock in production. --- src/iocore/cache/CacheShm.cc | 22 +++++++++++++++++++ src/iocore/cache/CacheShm.h | 5 +++++ .../cache/unit_tests/test_CacheShmShutdown.cc | 2 ++ 3 files changed, 29 insertions(+) diff --git a/src/iocore/cache/CacheShm.cc b/src/iocore/cache/CacheShm.cc index a586a58feb1..8e7cdc39c28 100644 --- a/src/iocore/cache/CacheShm.cc +++ b/src/iocore/cache/CacheShm.cc @@ -155,6 +155,11 @@ CacheShm::detach_stripe(char *) { } +void +CacheShm::release_for_test() +{ +} + #else namespace @@ -929,4 +934,21 @@ CacheShm::detach_stripe(char *raw_dir) g_pointers.erase(it); } +void +CacheShm::release_for_test() +{ + std::scoped_lock lk{g_table_mutex}; + if (g_control != nullptr) { + munmap(g_control, CONTROL_SIZE); + g_control = nullptr; + } + // Closing the fd is what releases the control flock, so the next initialize() in this process is not refused as a + // concurrent attach. + g_control_fd = ats_scoped_fd{}; + g_control_name.clear(); + std::memset(g_entry_claimed, 0, sizeof(g_entry_claimed)); + g_claims_this_run = 0; + _mode = Mode::Disabled; +} + #endif // TS_USE_CACHE_SHM diff --git a/src/iocore/cache/CacheShm.h b/src/iocore/cache/CacheShm.h index b9bb579f1fd..751f508bfd6 100644 --- a/src/iocore/cache/CacheShm.h +++ b/src/iocore/cache/CacheShm.h @@ -81,6 +81,11 @@ class CacheShm /// Backs the concurrent-attach owner-liveness guard. Exposed for unit testing. static bool process_is_alive(int pid); + /// Drops the process-wide segment state so a unit test can exercise a second start in one process. Stands in for process + /// exit, which is the only thing that does this in production: the control flock is held for the process lifetime, and + /// where the platform honors it a second initialize() would otherwise be refused by the concurrent-attach guard. + static void release_for_test(); + private: static Mode _mode; }; diff --git a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc index b79d9dea89f..ab5cc1c49a1 100644 --- a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc +++ b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc @@ -204,6 +204,7 @@ TEST_CASE("An untrusted stripe entry is recreated instead of attached", "[cache] const uint32_t count_before = control_stripe_count(); // Baseline: an unmarked entry is reused, so the sentinel survives the restart. + CacheShm::release_for_test(); REQUIRE(enable_shm()); REQUIRE(CacheShm::mode() == CacheShm::Mode::AttachExisting); char *reattached = CacheShm::attach_or_create_stripe(key, dir_size); @@ -216,6 +217,7 @@ TEST_CASE("An untrusted stripe entry is recreated instead of attached", "[cache] CacheShm::detach_stripe(reattached); // Same key, same size, entry still present -- but marked, so a fresh (zero-filled) segment must come back instead. + CacheShm::release_for_test(); REQUIRE(enable_shm()); REQUIRE(CacheShm::mode() == CacheShm::Mode::AttachExisting); char *fresh = CacheShm::attach_or_create_stripe(key, dir_size); From d46b3dd5995131aa311845deaaaf109ee2d59ee3 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 4 Aug 2026 08:56:12 +0900 Subject: [PATCH 4/7] cache: always write the on-disk dir at shm shutdown Skipping the write for a shm-backed stripe looked free -- the segment is already current and is attached directly next start -- but the on-disk copy is the only thing the fallback has, and recover_data() cannot always rebuild from it. handle_recover_from_data() returns without scanning the data region when the on-disk header still has sync_serial == 0, so an empty directory is accepted as-is. A stripe filled and cleanly shut down before the first periodic dir sync (60 s by default) is exactly that case: nothing had written the on-disk dir, so the next start that cannot use the segment found an empty directory and lost every object. cache_shm_dir_invalid caught it as a 502 against its deliberately absent origin, once the poked segment was correctly rejected. Only start time is what this feature set out to improve, so the shutdown write costs what it did before and the fallback stays recoverable. --- .../shm-fast-restart.en.rst | 38 +++++++++++-------- src/iocore/cache/StripeSM.cc | 18 ++------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst index 62bd47a08f1..d3b40b10e6a 100644 --- a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst +++ b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst @@ -474,22 +474,28 @@ the directory structurally rather than trusting ``clean_shutdown`` alone -- see is harmless either way: the read path checks ``Doc`` magic and key before serving, so a stale entry resolves to a miss. -Skipping the on-disk directory write ------------------------------------- - -For a shared-memory-backed stripe, ``StripeSM::shutdown`` still flushes -the aggregation buffer (so pending *content* reaches disk) but then **skips the -on-disk directory write** entirely: - -.. code-block:: text - - Dir : shm-backed, skipping on-disk directory write - -The shared segment is already the durable copy of the directory and is attached -directly next start, so writing the A/B copies to disk would be pure waste. The -trade-off is deliberate and safe: if the segment is later dropped for any -reason, the on-disk A/B copies plus ``StripeSM::recover_data`` reconcile -the tail -- the same path an unclean restart already takes. +The on-disk directory is still written +-------------------------------------- + +``StripeSM::shutdown`` writes the on-disk A/B directory copy for a +shared-memory-backed stripe exactly as it does without this feature. Skipping it +looks like an easy win -- the segment is already the current copy and is attached +directly next start, so the write appears to be pure waste -- but it is not safe. + +The on-disk copy is the *only* thing the fallback has whenever the segment is +dropped, and ``StripeSM::recover_data`` cannot always reconstruct what is missing +from it. When the on-disk header still carries ``sync_serial == 0``, +``handle_recover_from_data`` returns straight to ``handle_recover_write_dir`` +without scanning the data region at all, so an empty directory is accepted as-is. +A stripe filled and cleanly shut down before the first periodic dir sync +(:ts:cv:`proxy.config.cache.dir.sync_frequency`, 60 s by default) is exactly that +case: skipping the shutdown write leaves ``sync_serial == 0`` on disk, and the +next start that cannot use the segment finds an empty directory and silently +loses every object. The ``cache_shm_dir_invalid`` autest covers this. + +So the shutdown write stays. It costs what it cost before the feature existed, +and it buys the guarantee that the fallback path is always recoverable. Only +*start* time is what this feature set out to improve. Flush failure at shutdown ------------------------- diff --git a/src/iocore/cache/StripeSM.cc b/src/iocore/cache/StripeSM.cc index e76745fe74b..d26813f8bee 100644 --- a/src/iocore/cache/StripeSM.cc +++ b/src/iocore/cache/StripeSM.cc @@ -1349,14 +1349,11 @@ StripeSM::shutdown(EThread *shutdown_thread) return; } - bool shm_backed = CacheShm::is_shm_pointer(this->directory.raw_dir); - - // aggWriteDone advances write_pos again once we drop the mutex, so the shm header is not final; fall through to the - // on-disk write instead, which recover_data() reconciles next start. - if (shm_backed && this->is_io_in_progress()) { + // aggWriteDone advances write_pos again once we drop the mutex, so the shm header is not final; the on-disk write below + // plus recover_data() next start reconcile it. + if (CacheShm::is_shm_pointer(this->directory.raw_dir) && this->is_io_in_progress()) { Dbg(dbg_ctl_cache_dir_sync, "Dir %s: AIO write in flight -- invalidating shm copy, syncing dir to disk", this->hash_text.get()); CacheShm::invalidate_stripe_directory(this->directory.raw_dir); - shm_backed = false; } size_t dirlen = this->dirlen(); @@ -1375,10 +1372,8 @@ StripeSM::shutdown(EThread *shutdown_thread) if (!this->flush_aggregate_write_buffer(this->fd)) { // Mark rather than lean on the unquiesced cursor the failure leaves behind: the event system is still up, so a later // aggWriteDone or agg_wrap() can re-equalize agg_pos and write_pos and the gate would let this segment through. - // Still sync: an entry for unwritten data reads as a miss, but skipping loses every insert since the last sync. Error("Dir %s: aggregation buffer flush failed during shutdown; syncing the directory to disk", this->hash_text.get()); CacheShm::invalidate_stripe_directory(this->directory.raw_dir); - shm_backed = false; } } @@ -1392,13 +1387,6 @@ StripeSM::shutdown(EThread *shutdown_thread) CHECK_DIR(d); - // Pure waste for a shm-backed dir: it is already current in the segment and is attached directly next start. If the - // segment is later dropped, the on-disk A/B copies plus recover_data() reconcile the tail. - if (shm_backed) { - Note("Dir %s: shm-backed, skipping on-disk directory write", this->hash_text.get()); - return; - } - size_t B = this->directory.header->sync_serial & 1; off_t start = this->skip + (B ? dirlen : 0); B = pwrite(this->fd, this->directory.raw_dir, dirlen, start); From 837deb370b7ea0320aea36f9edd5bed4372e9ef8 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 7 Aug 2026 07:40:29 +0900 Subject: [PATCH 5/7] cache: harden the shm trust gates and ownership guard Review of the fast-restart path found four ways a shared-memory segment could be trusted, or cleared, when it should not be. The attach gate bounded directory links but neither a live entry's offset -- which CacheVC::handleRead turns into a negative, so huge unsigned, read length -- nor the free list's structure, where an in-range cycle lets freelist_pop write a link over a live entry's tag bits. Clean shutdown cleared owner_pid while event threads were still writing, which on a platform where flock is a no-op is the only guard against a concurrent attach. And traffic_ctl swept a control segment smaller than this build's before checking for a live owner, so a newer build could unlink a running older build's segments -- the upgrade case the frozen header exists to support. The entry bound is on where an entry starts, not on its extent: dir_approx_size rounds up, so the last object in a stripe legitimately overhangs the stripe end, which is what handleRead's truncation is for. --- .../shm-fast-restart.en.rst | 51 +++++++-- include/shared/cache_shm/Layout.h | 5 +- include/shared/cache_shm/Purge.h | 59 +++++++---- src/iocore/cache/CacheShm.cc | 8 +- src/iocore/cache/Stripe.cc | 41 ++++++- src/iocore/cache/Stripe.h | 1 + src/iocore/cache/unit_tests/test_CacheShm.cc | 100 ++++++++++++++++-- .../cache/unit_tests/test_CacheShmShutdown.cc | 75 +++++++++++++ .../cache/cache_shm_bad_disk_dropped.test.py | 8 +- .../cache/cache_shm_concurrent_attach.test.py | 2 +- .../cache/cache_shm_fast_restart.test.py | 4 +- .../cache/cache_shm_storage_mismatch.test.py | 4 +- .../gold/cache_shm_state_after_shutdown.gold | 2 +- 13 files changed, 306 insertions(+), 54 deletions(-) diff --git a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst index d3b40b10e6a..5ce4af1ad7e 100644 --- a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst +++ b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst @@ -157,8 +157,9 @@ up to ``MAX_STRIPES`` (256) ``cache_shm::StripeEntry`` rows. A other times, including throughout a running process, so a crash leaves it ``0``. * - ``owner_pid`` - - PID of the process currently mapping the segment read-write, or ``0`` - when none. Backs the concurrent-attach guard. + - PID of the process that took the segment, or ``0`` when none. Backs the + concurrent-attach guard, so it is held until that process exits rather + than cleared at clean shutdown. * - ``stripe_count`` - High-water mark of used rows in ``stripes[]``. * - ``stripes[]`` @@ -305,9 +306,13 @@ is therefore based on ownership, with two layers: (``LockResult::Unsupported``). There, the guard falls back to the recorded ``owner_pid``: if it names a live process other than ourselves (``CacheShm::process_is_alive``, via ``kill(pid, 0)``), the new - process disables shared memory. A clean shutdown clears ``owner_pid`` to - ``0``; a crash leaves a stale pid, but a crash also leaves - ``clean_shutdown = 0``, so the segment is dropped by that gate anyway. + process disables shared memory. The pid is held until the owner exits and is + *not* cleared at clean shutdown, because ``mark_clean_shutdown`` runs while the + event threads are still writing (see `Wiring`_): on a platform with no lock, + clearing it there is what would let a second process attach into that window. + A crash leaves a stale pid, but a crash also leaves ``clean_shutdown = 0``, so + the segment is dropped by that gate anyway; a pid that has been recycled by an + unrelated process costs the next start its fast restart and nothing more. A symmetric check guards the ``CreateFresh`` path: after creating the fresh control segment, ``initialize`` takes the lock, and if it lost a creation race @@ -402,6 +407,24 @@ and the shape of the entry graph before the attach: hold fewer entries than those bits can spell -- the ``head`` bit alone is 8192, so every stripe under roughly 65 MB failed validation and silently gave up fast restart; +* every *in-use* entry starts inside the stripe, i.e. ``vol_offset(e) < skip + + len`` -- the same invariant ``Directory::insert()`` asserts. ``dir_valid()`` + cannot stand in for it: an in-phase entry is bounded above by ``write_pos``, but + an out-of-phase one is bounded from *below* only (``vol_out_of_phase_valid``), so + a torn 40-bit offset passes. ``CacheVC::handleRead`` then computes an + out-of-stripe ``aio_offset``, and its end truncation subtracts past zero into a + ``size_t`` ``aio_nbytes`` of roughly 2^64 while the buffer it sizes from the same + value comes back at the *smallest* index -- a read far larger than its + destination. Note the bound is on where the entry *starts*, not on its extent: + ``dir_approx_size`` rounds up, so the last object in a stripe legitimately + overhangs ``skip + len``, which is precisely what that truncation exists for; +* each segment's free list is walked structurally, not just bounds-checked: at most + one node per entry, every node empty, and ``prev`` pointing back at the node + nearer the head. An in-range cycle such as ``A -> B -> A`` satisfies every + per-entry check above and is invisible to ``check_segment()``, which walks bucket + chains. Attached, it lets ``freelist_pop()`` hand out an entry twice and write a + free-list link over the second holder's ``tag``/``phase``/``head``/``pinned`` + word, until the ``dir_offset(e)`` guard in ``freelist_pop`` resets the segment; * ``Directory::check_segment()`` passes for the segment: bucket chain lengths, no chained-but-empty entries, no chain loops. This is the ``CHECK_DIR`` walk that is otherwise debug-only, and it is what turns a torn directory into a rebuild rather @@ -461,10 +484,11 @@ On a clean exit, ``AutoStopCont::mainEvent`` calls ``sync_cache_dir_on_shutdown()`` whenever the cache is initialized. ``sync_cache_dir_on_shutdown`` snapshots every stripe (taking each stripe mutex, which excludes writers *concurrent with* the snapshot), and only then calls -``CacheShm::mark_clean_shutdown``, which sets ``clean_shutdown = 1``, clears -``owner_pid`` to ``0``, and ``msync``\ s the header. When the feature is -disabled, ``mark_clean_shutdown`` is a no-op (there is no control segment), so -the shutdown path is unchanged for a stock |TS|. +``CacheShm::mark_clean_shutdown``, which sets ``clean_shutdown = 1`` and +``msync``\ s the header. ``owner_pid`` is deliberately left alone; see +`Concurrent-attach guard`_. When the feature is disabled, +``mark_clean_shutdown`` is a no-op (there is no control segment), so the shutdown +path is unchanged for a stock |TS|. It does not stop *future* writers: this runs before ``shut_down_event_system()`` and the main thread exits without joining the event threads, so a stripe can @@ -677,6 +701,15 @@ header-only form (:ts:git:`include/shared/cache_shm/Layout.h` and first), so it cannot orphan a running instance's fast restart. This is the on-demand equivalent of ``purge_stale_on_start``. + The owner check comes *before* any branch on the segment's size. A segment + smaller than this build's ``CacheShmControl`` is an *older* build's, and that + build may still be running -- exactly the upgrade case the frozen header exists + to make legible. ``flock`` needs only the fd, so it costs nothing to take first; + the frozen header prefix is then mapped for the ``owner_pid`` backstop, mapping + only what the object actually holds, since a mapping past an object's last page + faults on access. A segment too short to hold even the frozen header was written + by no build of ours, so there is no owner to protect and it is swept. + .. _cache-shm-configuration: Configuration diff --git a/include/shared/cache_shm/Layout.h b/include/shared/cache_shm/Layout.h index bda3a8539d4..fa6a8e65228 100644 --- a/include/shared/cache_shm/Layout.h +++ b/include/shared/cache_shm/Layout.h @@ -70,8 +70,9 @@ struct CacheShmControl { uint64_t storage_signature; ///< storage.yaml fingerprint uint8_t clean_shutdown; ///< 0 = dirty, 1 = clean uint8_t pad1[3]; - int32_t owner_pid; ///< PID holding the exclusive lock; 0 when none. Backs the - ///< concurrent-attach guard. Cleared on clean shutdown. + int32_t owner_pid; ///< PID of the process that took the segment; 0 when none. Backs the + ///< concurrent-attach guard, so it is held until the owner exits, not + ///< cleared at clean shutdown; the next start tests it for liveness. uint32_t stripe_count; uint32_t pad2; StripeEntry stripes[MAX_STRIPES]; diff --git a/include/shared/cache_shm/Purge.h b/include/shared/cache_shm/Purge.h index f48ffa908e7..1b170b385e1 100644 --- a/include/shared/cache_shm/Purge.h +++ b/include/shared/cache_shm/Purge.h @@ -243,6 +243,36 @@ purge_segments(const std::string &prefix) return report; } report.segment_size = static_cast(sb.st_size); + + // Before any size branch: flock needs only the fd, and a segment too small for *our* layout can still be a live older + // build's -- unlinking it would strip the names out from under the very upgrade the frozen header exists to support. + const LockResult lock = try_lock_control(fd); + + // Map only what is really there. A foreign build's segment may be shorter than CONTROL_SIZE, and mapping past the last + // page of the object faults on access; the frozen header prefix is all the owner guard needs. + const std::size_t map_len = std::min(static_cast(sb.st_size), CONTROL_SIZE); + void *addr = nullptr; + if (map_len >= CONTROL_HEADER_SIZE) { + addr = ::mmap(nullptr, map_len, PROT_READ, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + report.sys_errno = errno; + report.outcome = PurgeOutcome::MapFailed; + return report; + } + } + const auto *ctrl = static_cast(addr); + // Too short to even hold the frozen header: no build of ours wrote it, so there is no owner to protect. + const bool magic_ok = ctrl != nullptr && std::memcmp(ctrl->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)) == 0; + + if (lock == LockResult::HeldByOther || (lock == LockResult::Unsupported && magic_ok && process_is_alive(ctrl->owner_pid))) { + report.owner_pid = magic_ok ? ctrl->owner_pid : 0; + report.outcome = PurgeOutcome::OwnedByLive; + if (addr != nullptr) { + ::munmap(addr, map_len); + } + return report; + } + if (static_cast(sb.st_size) < CONTROL_SIZE) { // Too small to hold this build's header/table: there is no table to walk, so // sweep the stripe name space and unlink the control object itself. @@ -250,40 +280,23 @@ purge_segments(const std::string &prefix) sweep_stripe_name_space(); int e = ::shm_unlink(report.control_name.c_str()) == 0 ? 0 : errno; report.unlinked.push_back({report.control_name, true, e}); + if (addr != nullptr) { + ::munmap(addr, map_len); + } return report; } // Larger than this build's page-rounded CONTROL_SIZE means a build with a different // sizeof(CacheShmControl) wrote it. The frozen header prefix is still readable (so - // the owner guard below applies), but stripes[] may have a different stride entirely, + // the owner guard above applies), but stripes[] may have a different stride entirely, // so its names must not drive shm_unlink. - const bool own_size = is_own_control_size(static_cast(sb.st_size)); - - void *addr = ::mmap(nullptr, CONTROL_SIZE, PROT_READ, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - report.sys_errno = errno; - report.outcome = PurgeOutcome::MapFailed; - return report; - } - - const auto *ctrl = static_cast(addr); - const bool magic_ok = std::memcmp(ctrl->magic, CACHE_SHM_MAGIC, sizeof(CACHE_SHM_MAGIC)) == 0; - - const LockResult lock = try_lock_control(fd); - if (lock == LockResult::HeldByOther || (lock == LockResult::Unsupported && magic_ok && process_is_alive(ctrl->owner_pid))) { - report.owner_pid = magic_ok ? ctrl->owner_pid : 0; - report.outcome = PurgeOutcome::OwnedByLive; - ::munmap(addr, CONTROL_SIZE); - return report; - } - - if (magic_ok && own_size) { + if (magic_ok && is_own_control_size(static_cast(sb.st_size))) { unlink_table_stripes(prefix, ctrl, report.unlinked); } else { // Bad magic, or a foreign sizeof(CacheShmControl): the table cannot be walked. sweep_stripe_name_space(); } - ::munmap(addr, CONTROL_SIZE); + ::munmap(addr, map_len); int e = ::shm_unlink(report.control_name.c_str()) == 0 ? 0 : errno; report.unlinked.push_back({report.control_name, true, e}); diff --git a/src/iocore/cache/CacheShm.cc b/src/iocore/cache/CacheShm.cc index 8e7cdc39c28..1fab3327213 100644 --- a/src/iocore/cache/CacheShm.cc +++ b/src/iocore/cache/CacheShm.cc @@ -885,10 +885,10 @@ CacheShm::mark_clean_shutdown() } Note("cache shm: marking clean shutdown"); g_control->clean_shutdown = 1; - // Clear owner_pid so the next start's liveness backstop does not mistake our - // (exiting) PID for a live owner. The flock is still held until exit, so a - // concurrent starter is still correctly refused during the shutdown window. - g_control->owner_pid = 0; + // owner_pid stays until the process exits. Event threads are still running here (see sync_cache_dir_on_shutdown), and + // where flock is a no-op -- macOS, FreeBSD -- it is the only thing that stops a starting process from attaching to + // directories we may still be writing. A stale PID only costs the next start its fast restart; a concurrent attach + // costs correctness. msync(g_control, CONTROL_SIZE, MS_SYNC); } diff --git a/src/iocore/cache/Stripe.cc b/src/iocore/cache/Stripe.cc index 261b48a202b..cf647c28044 100644 --- a/src/iocore/cache/Stripe.cc +++ b/src/iocore/cache/Stripe.cc @@ -220,11 +220,23 @@ Stripe::_shm_directory_is_valid() if (dir_next(e) >= segment_entries) { return false; } - if (dir_is_empty(e) && dir_prev(e) >= segment_entries) { + if (dir_is_empty(e)) { + if (dir_prev(e) >= segment_entries) { + return false; + } + continue; + } + // Same invariant Directory::insert() asserts. dir_valid() cannot stand in for it: an out-of-phase entry is bounded + // from below only, and CacheVC::handleRead() turns an out-of-stripe offset into a negative (so huge) read length. + if (this->vol_offset(e) >= data_hi) { return false; } } + if (!this->_shm_freelist_is_valid(s, seg, segment_entries)) { + return false; + } + if (!this->directory.check_segment(s)) { return false; } @@ -233,6 +245,33 @@ Stripe::_shm_directory_is_valid() return true; } +// The per-entry bounds above only prove each link points inside the segment; an in-range cycle or a stale prev link +// still passes, and Directory::check_segment() walks the bucket chains, not the free list. +bool +Stripe::_shm_freelist_is_valid(int s, Dir *seg, int64_t segment_entries) +{ + int64_t node = this->directory.header->freelist[s]; + int64_t prev = 0; + int64_t steps = 0; + + while (node != 0) { + if (node >= segment_entries || ++steps > segment_entries) { + return false; + } + Dir *e = dir_in_seg(seg, node); + + // Every producer (Directory::free_entry, delete_entry, freelist_pop, unlink_from_freelist) clears the entry and + // leaves prev pointing back at the node nearer the head, so a mismatch means the list was left half-updated. + if (!dir_is_empty(e) || dir_prev(e) != prev) { + return false; + } + prev = node; + node = dir_next(e); + } + + return true; +} + // coverity[exn_spec_violation] - ink_assert aborts (doesn't throw), Dbg is exception-safe Stripe::~Stripe() { diff --git a/src/iocore/cache/Stripe.h b/src/iocore/cache/Stripe.h index a5656c0a0eb..e863e8ace47 100644 --- a/src/iocore/cache/Stripe.h +++ b/src/iocore/cache/Stripe.h @@ -158,6 +158,7 @@ class Stripe void _init_data(off_t store_block_size, int avg_obj_size = -1); void _init_data_internal(int avg_obj_size = -1); // Defaults to cache_config_min_average_object_size; void _init_directory(std::size_t directory_size, int header_size, int footer_size); + bool _shm_freelist_is_valid(int s, Dir *seg, int64_t segment_entries); }; inline uint32_t diff --git a/src/iocore/cache/unit_tests/test_CacheShm.cc b/src/iocore/cache/unit_tests/test_CacheShm.cc index c5156feada0..790e4841845 100644 --- a/src/iocore/cache/unit_tests/test_CacheShm.cc +++ b/src/iocore/cache/unit_tests/test_CacheShm.cc @@ -32,12 +32,15 @@ #include "tscore/ink_config.h" #include "tscore/ink_memory.h" +#include #include #include #include #include +#include #include +#include #include // Required by the shared test harness (main.cc). @@ -232,9 +235,10 @@ namespace // A prefix of our own so these tests can never touch a real instance's segments. constexpr const char *PURGE_PREFIX_WORD = "atspurgetest"; -// Valid magic, no owner, one claimed stripe. False if shm is unavailable here. +// Valid magic, one claimed stripe, and `owner_pid` as given. `size` may be short of CONTROL_SIZE -- an older build with a +// smaller stripe table -- so only what exists is mapped. False if shm is unavailable here. bool -plant_control_segment(const std::string &prefix, std::size_t size) +plant_control_segment(const std::string &prefix, std::size_t size, int32_t owner_pid = 0) { const std::string name = cache_shm::control_segment_name(prefix); shm_unlink(name.c_str()); @@ -247,20 +251,21 @@ plant_control_segment(const std::string &prefix, std::size_t size) shm_unlink(name.c_str()); return false; } - void *addr = mmap(nullptr, cache_shm::CONTROL_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + const std::size_t map_len = std::min(size, cache_shm::CONTROL_SIZE); + void *addr = mmap(nullptr, map_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); if (addr == MAP_FAILED) { shm_unlink(name.c_str()); return false; } auto *ctrl = static_cast(addr); - std::memset(ctrl, 0, cache_shm::CONTROL_SIZE); + std::memset(ctrl, 0, map_len); std::memcpy(ctrl->magic, cache_shm::CACHE_SHM_MAGIC, sizeof(cache_shm::CACHE_SHM_MAGIC)); ctrl->schema_version = cache_shm::CACHE_SHM_SCHEMA_VERSION; - ctrl->owner_pid = 0; // no live owner, so purge is allowed to proceed + ctrl->owner_pid = owner_pid; ctrl->stripe_count = 1; // Deliberately unnamed: a foreign build's stripes[] may have another stride, so a correct purge cannot read this. - munmap(addr, cache_shm::CONTROL_SIZE); + munmap(addr, map_len); return true; } @@ -288,6 +293,21 @@ segment_exists(const std::string &name) return true; } +// The kernel rounds an shm object up to a page, so a segment shorter than CONTROL_SIZE is not representable everywhere: +// Apple Silicon's 16 KB page already exceeds it. -1 if the segment is gone. +long long +segment_size(const std::string &name) +{ + int fd = shm_open(name.c_str(), O_RDONLY, 0); + if (fd < 0) { + return -1; + } + struct stat sb; + const int rc = fstat(fd, &sb); + close(fd); + return rc < 0 ? -1 : static_cast(sb.st_size); +} + } // namespace // A foreign sizeof(CacheShmControl) leaves the frozen header readable but the stripe table not. Purging must fall back to @@ -340,4 +360,72 @@ TEST_CASE("CacheShm purge walks the table when the control layout is ours", "[ca shm_unlink(cache_shm::control_segment_name(prefix).c_str()); } +// A segment shorter than our CacheShmControl was written by an *older* build, which may still be running. The frozen +// header is there so a newer traffic_ctl can recognise that owner, not so it can unlink the names out from under it. +TEST_CASE("CacheShm purge refuses a smaller foreign control segment with a live owner", "[cache][shm]") +{ + const std::string prefix = cache_shm::normalize_name_prefix(PURGE_PREFIX_WORD); + const std::string control = cache_shm::control_segment_name(prefix); + const std::string stripe_name = cache_shm::stripe_segment_name(prefix, 0); + + // Past the frozen header so the owner is readable, short of CONTROL_SIZE so the table cannot be walked. + if (!plant_control_segment(prefix, cache_shm::CONTROL_HEADER_SIZE + 64, static_cast(getpid()))) { + WARN("shm unavailable in this environment; skipping"); + return; + } + if (segment_size(control) >= static_cast(cache_shm::CONTROL_SIZE)) { + WARN("shm objects round up past CONTROL_SIZE here; a smaller foreign segment is not representable"); + shm_unlink(control.c_str()); + return; + } + REQUIRE(plant_stripe_segment(stripe_name)); + + // Hold the lock too, so the refusal is asserted on both kinds of platform: flock decides where it is honoured (a second + // open file description conflicts even within one process), owner_pid where it is not. + int held = shm_open(control.c_str(), O_RDONLY, 0); + REQUIRE(held >= 0); + (void)::flock(held, LOCK_EX | LOCK_NB); + + const cache_shm::PurgeReport report = cache_shm::purge_segments(prefix); + + CHECK(report.outcome == cache_shm::PurgeOutcome::OwnedByLive); + CHECK(report.unlinked.empty()); + CHECK(segment_exists(stripe_name)); + CHECK(segment_exists(control)); + + close(held); + shm_unlink(stripe_name.c_str()); + shm_unlink(control.c_str()); +} + +// The counterpart: with no live owner the same short segment must still be cleared, or an operator cannot recover from a +// stale one left by a build that is gone. +TEST_CASE("CacheShm purge clears a smaller foreign control segment with no owner", "[cache][shm]") +{ + const std::string prefix = cache_shm::normalize_name_prefix(PURGE_PREFIX_WORD); + const std::string control = cache_shm::control_segment_name(prefix); + const std::string stripe_name = cache_shm::stripe_segment_name(prefix, 0); + + if (!plant_control_segment(prefix, cache_shm::CONTROL_HEADER_SIZE + 64)) { + WARN("shm unavailable in this environment; skipping"); + return; + } + if (segment_size(control) >= static_cast(cache_shm::CONTROL_SIZE)) { + WARN("shm objects round up past CONTROL_SIZE here; a smaller foreign segment is not representable"); + shm_unlink(control.c_str()); + return; + } + REQUIRE(plant_stripe_segment(stripe_name)); + + const cache_shm::PurgeReport report = cache_shm::purge_segments(prefix); + + CHECK(report.outcome == cache_shm::PurgeOutcome::TooSmall); + CHECK(report.table_untrusted); + CHECK_FALSE(segment_exists(stripe_name)); + CHECK_FALSE(segment_exists(control)); + + shm_unlink(stripe_name.c_str()); + shm_unlink(control.c_str()); +} + #endif // TS_USE_CACHE_SHM diff --git a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc index ab5cc1c49a1..ff5c53787dc 100644 --- a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc +++ b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc @@ -318,6 +318,81 @@ TEST_CASE("A small stripe's directory validates with in-use entries", "[cache][s unlink_test_segments(); } +// A live entry's offset feeds CacheVC::handleRead, where an offset past the stripe truncates the read length to a negative +// -- so huge unsigned -- value. dir_valid() is no help: an out-of-phase entry is bounded from below only. +TEST_CASE("A directory entry starting past the stripe fails the attach gate", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + disk.hw_sector_size = 512; + attach_tmpfile_to_stripe(stripe); + stripe.clear_dir(); + REQUIRE(stripe.shm_directory_is_valid()); + + // The last entry that still *starts* inside the stripe has to pass: its extent may legitimately overhang the end, + // which is exactly what handleRead's truncation is for, so bounding the extent would reject healthy directories. + Dir *e = stripe.directory.dir; + dir_set_offset(e, stripe.offset_to_vol_offset(stripe.skip + stripe.len) - 1); + dir_set_head(e, 1); + REQUIRE(stripe.vol_offset(e) < stripe.skip + stripe.len); + CHECK(stripe.shm_directory_is_valid()); + + // One block on, the read would start at or past the end of the stripe. + dir_set_offset(e, dir_offset(e) + 1); + REQUIRE(stripe.vol_offset(e) >= stripe.skip + stripe.len); + CHECK_FALSE(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + +// A cycle in a segment's free list is made of in-range links, so the per-entry bounds pass, and check_segment() walks the +// bucket chains rather than the free list. Left unchecked, freelist_pop() writes a link over a live entry's tag bits. +TEST_CASE("A cyclic segment free list fails the attach gate", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + disk.hw_sector_size = 512; + attach_tmpfile_to_stripe(stripe); + stripe.clear_dir(); + REQUIRE(stripe.shm_directory_is_valid()); + + Dir *seg = stripe.directory.get_segment(0); + const uint16_t head = stripe.directory.header->freelist[0]; + Dir *next = dir_in_seg(seg, dir_next(dir_in_seg(seg, head))); + REQUIRE(dir_next(dir_in_seg(seg, head)) != 0); + + // Point the head's successor back at the head. Both links stay inside the segment, and no bucket chain is touched. + dir_set_next(next, head); + + CHECK_FALSE(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + // What the flush-failure path actually leans on. A real short write leaves write_pos in range with agg_pos ahead of it, so // the quiesced-cursor check has to be what rejects the segment -- not the range check, which only a synthetic write_pos // trips. diff --git a/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py b/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py index 7b7921783be..168fb5593af 100644 --- a/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py +++ b/tests/gold_tests/cache/cache_shm_bad_disk_dropped.test.py @@ -169,7 +169,9 @@ def _dump_shm_state(self): # The gold masks the run-specific names, the ABI/storage hashes, and the # page-rounded sizes with the `` wildcard, so what is asserted literally # is the meaningful state: valid magic/schema, clean_shutdown=1, - # stripe_count=2, and both stripe segments present. + # stripe_count=2, both stripe segments present, and a retained owner_pid + # that reads as stale (a clean shutdown keeps the pid so a platform + # without flock cannot attach into the shutdown window). tr = Test.AddTestRun('Dump shm control state after ts1 clean shutdown') # Use ts1's Env: it has been started, so the per-instance bin dir is on # PATH (ts2's Env only gains it once ts2 starts, which is the next step). @@ -197,8 +199,8 @@ def _verify_survivor_attach_and_reclaim(self): def _clean_shutdown_ts2(self): # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a - # segment a live traffic_server still owns, so the segments must be - # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + # segment a live traffic_server still owns, so the owner must be gone + # (its retained owner_pid then reads as dead) before cleanup runs. tr = Test.AddTestRun('Drain and clean-shutdown ts2') tr.Processes.Default.Env = self.ts2.Env tr.Processes.Default.Command = ( diff --git a/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py b/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py index 90453312708..cfbe544de95 100644 --- a/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py +++ b/tests/gold_tests/cache/cache_shm_concurrent_attach.test.py @@ -176,7 +176,7 @@ def _cleanup_shm(self): def run(self): self._start_owner() self._start_second_refused() - # Stop the non-owner first, then the owner (clears owner_pid), then clear. + # Stop the non-owner first, then the owner, then clear: its retained owner_pid reads as dead once it exits. self._clean_shutdown(self.ts2, 'shmc_ts2') self._clean_shutdown(self.ts1, 'shmc_ts1') self._cleanup_shm() diff --git a/tests/gold_tests/cache/cache_shm_fast_restart.test.py b/tests/gold_tests/cache/cache_shm_fast_restart.test.py index 520b2b00986..b4269fb9f8a 100644 --- a/tests/gold_tests/cache/cache_shm_fast_restart.test.py +++ b/tests/gold_tests/cache/cache_shm_fast_restart.test.py @@ -214,8 +214,8 @@ def _start_ts2(self): def _clean_shutdown_ts2(self): # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a - # segment a live traffic_server still owns, so the segments must be - # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + # segment a live traffic_server still owns, so the owner must be gone + # (its retained owner_pid then reads as dead) before cleanup runs. tr = Test.AddTestRun('Drain and clean-shutdown ts2') tr.Processes.Default.Env = self.ts2.Env tr.Processes.Default.Command = ( diff --git a/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py b/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py index ad4262f8052..ff037fb29ff 100644 --- a/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py +++ b/tests/gold_tests/cache/cache_shm_storage_mismatch.test.py @@ -178,8 +178,8 @@ def _verify_partial_attach_and_reclaim(self): def _clean_shutdown_ts2(self): # Stop ts2 before clearing the shm: `cache shm clear` refuses to unlink a - # segment a live traffic_server still owns, so the segments must be - # ownerless (clean_shutdown clears owner_pid) before cleanup runs. + # segment a live traffic_server still owns, so the owner must be gone + # (its retained owner_pid then reads as dead) before cleanup runs. tr = Test.AddTestRun('Drain and clean-shutdown ts2') tr.Processes.Default.Env = self.ts2.Env tr.Processes.Default.Command = ( diff --git a/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold b/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold index 5a981414f4f..925483b2c5e 100644 --- a/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold +++ b/tests/gold_tests/cache/gold/cache_shm_state_after_shutdown.gold @@ -5,7 +5,7 @@ Control segment: `` abi_hash: 0x`` storage_sig: 0x`` clean_shutdown: 1 (clean) - owner_pid: 0 (none -- not currently attached) + owner_pid: `` (stale -- owner no longer running) stripe_count: 2 Stripes: From 45109d483b3b387649393ff6a4a243cf3598567e Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 7 Aug 2026 09:56:41 +0900 Subject: [PATCH 6/7] cache: prove shm segment membership at attach A walk from the free-list head cannot see an entry that Directory::insert unlinked but never filled, so a shutdown torn in that window published a directory whose stale prev/next the next insert would write through. Require every entry to be reached exactly once, as a bucket root, an empty free-list node, or an in-use chain node, and delay the clean-shutdown mark until after the event system is down so fewer tears reach the gate at all. --- .../shm-fast-restart.en.rst | 87 ++++++++++++------ src/iocore/cache/CacheDir.cc | 8 +- src/iocore/cache/CacheShm.cc | 8 +- src/iocore/cache/CacheShm.h | 3 +- src/iocore/cache/Stripe.cc | 61 ++++++++++--- src/iocore/cache/Stripe.h | 3 +- .../cache/unit_tests/test_CacheShmShutdown.cc | 89 +++++++++++++++++++ src/traffic_server/traffic_server.cc | 8 ++ 8 files changed, 219 insertions(+), 48 deletions(-) diff --git a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst index 5ce4af1ad7e..14cb8f9f45b 100644 --- a/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst +++ b/doc/developer-guide/cache-architecture/shm-fast-restart.en.rst @@ -374,11 +374,12 @@ Validating a trusted segment The magic/version checks confirm the segment *looks like* a directory, but a stale-yet-magic-valid segment could still present offsets that would turn into -out-of-bounds disk I/O. And ``CacheShm::mark_clean_shutdown`` runs while the -event threads are still live (see `Shutdown`_), so a directory mutation torn at -process exit can sit behind a ``clean_shutdown = 1`` flag. -``Stripe::_shm_directory_is_valid`` therefore validates both the header fields -and the shape of the entry graph before the attach: +out-of-bounds disk I/O. And ``CacheShm::mark_clean_shutdown`` runs after +``TSSystemState::shut_down_event_system`` but joins no event thread (see +`Shutdown`_), so a directory mutation torn at process exit can still sit behind a +``clean_shutdown = 1`` flag. ``Stripe::_shm_directory_is_valid`` therefore +validates both the header fields and the shape of the entry graph before the +attach: * ``sector_size`` is non-zero and no larger than ``STORE_BLOCK_SIZE``; * ``write_pos``, ``last_write_pos`` and ``agg_pos`` all lie within the stripe's @@ -418,13 +419,39 @@ and the shape of the entry graph before the attach: destination. Note the bound is on where the entry *starts*, not on its extent: ``dir_approx_size`` rounds up, so the last object in a stripe legitimately overhangs ``skip + len``, which is precisely what that truncation exists for; -* each segment's free list is walked structurally, not just bounds-checked: at most - one node per entry, every node empty, and ``prev`` pointing back at the node - nearer the head. An in-range cycle such as ``A -> B -> A`` satisfies every - per-entry check above and is invisible to ``check_segment()``, which walks bucket - chains. Attached, it lets ``freelist_pop()`` hand out an entry twice and write a - free-list link over the second holder's ``tag``/``phase``/``head``/``pinned`` - word, until the ``dir_offset(e)`` guard in ``freelist_pop`` resets the segment; +* every entry in the segment is accounted for exactly once, by + ``_shm_segment_membership_is_valid``. This is a membership proof, not a + reachability walk: the free list is walked (every node empty, ``prev`` pointing + back at the node nearer the head), then every bucket chain is walked (every node + in use, no node a bucket root), each visit marked in a scratch bitmap, and + finally *every* index in the segment must have been marked. Anything reached + twice, and anything reached at all, fails. + + Reachability alone is not enough, because the state that matters is the one no + walk visits. ``Directory::insert()`` takes an empty row off the free list with + ``unlink_from_freelist()`` and only fills it several statements later; torn in + between, the row is empty, off the free list, not yet in any bucket chain, and + still holding the free-list ``prev``/``next`` it had. Every per-entry check above + passes, the remaining free list is self-consistent, and ``check_segment()`` walks + bucket chains. The next insert to scan that bucket finds the row empty, unlinks it + a second time, and writes ``dir_set_next`` into whatever its stale ``prev`` now + names -- truncating or cross-linking a live chain, or clobbering + ``header->freelist[s]`` when that ``prev`` is 0 -- and ``dir_set_prev`` into the + ``tag``/``phase``/``head``/``pinned`` word of whatever its stale ``next`` names. + The mirror-image tear, after the row is chained but before ``dir_assign_data`` + fills it, leaves an empty entry inside a bucket chain; it *is* reached exactly + once, which is why membership is paired with the empty/in-use polarity of where + each entry was reached. + + The same pass subsumes what a free-list-only walk caught: an in-range cycle such + as ``A -> B -> A`` satisfies every per-entry check and is invisible to + ``check_segment()``, and attached it lets ``freelist_pop()`` hand out an entry + twice. The invariant being proved is the one the live code maintains -- + ``init_segment()`` frees rows 1..``DIR_DEPTH``-1 of every bucket onto the free + list and never row 0, ``insert()`` moves an entry from the free list to a chain, + and ``delete_entry()`` moves it back -- so a clean segment always satisfies it, + and the cost is one ``segment_entries``-bit bitmap on a walk that was already + linear in the segment; * ``Directory::check_segment()`` passes for the segment: bucket chain lengths, no chained-but-empty entries, no chain loops. This is the ``CHECK_DIR`` walk that is otherwise debug-only, and it is what turns a torn directory into a rebuild rather @@ -483,20 +510,28 @@ Wiring On a clean exit, ``AutoStopCont::mainEvent`` calls ``sync_cache_dir_on_shutdown()`` whenever the cache is initialized. ``sync_cache_dir_on_shutdown`` snapshots every stripe (taking each stripe mutex, -which excludes writers *concurrent with* the snapshot), and only then calls -``CacheShm::mark_clean_shutdown``, which sets ``clean_shutdown = 1`` and -``msync``\ s the header. ``owner_pid`` is deliberately left alone; see -`Concurrent-attach guard`_. When the feature is disabled, -``mark_clean_shutdown`` is a no-op (there is no control segment), so the shutdown -path is unchanged for a stock |TS|. - -It does not stop *future* writers: this runs before ``shut_down_event_system()`` -and the main thread exits without joining the event threads, so a stripe can -still be written after its snapshot. That is why the fast-attach path re-validates -the directory structurally rather than trusting ``clean_shutdown`` alone -- see -`Validating a trusted segment`_. A late write that only lands a directory entry -is harmless either way: the read path checks ``Doc`` magic and key before serving, -so a stale entry resolves to a miss. +which excludes writers *concurrent with* the snapshot). Marking the segment clean +is deliberately **not** part of it: ``AutoStopCont::mainEvent`` calls +``CacheShm::mark_clean_shutdown`` itself, after ``shut_down_event_system()``, and +that is what sets ``clean_shutdown = 1`` and ``msync``\ s the header. ``owner_pid`` +is deliberately left alone; see `Concurrent-attach guard`_. When the feature is +disabled, ``mark_clean_shutdown`` is a no-op (there is no control segment), so the +shutdown path is unchanged for a stock |TS|. + +Ordering it after ``shut_down_event_system()`` matters because the mark is a +promise about the *whole* directory, not just the snapshot. Marked before it, any +event thread still running could be torn mid-``Directory::insert()`` and the +resulting half-linked directory would be published as trusted -- see +`Validating a trusted segment`_ for what that costs. + +It is not a hard barrier, though: ``shut_down_event_system()`` sets a flag and the +main thread exits without joining the event threads, so a straggler can still +write after the mark. That is why the fast-attach path re-validates the directory +structurally rather than trusting ``clean_shutdown`` alone. A late write that only +lands a directory *entry* is harmless either way: the read path checks ``Doc`` +magic and key before serving, so a stale entry resolves to a miss. A late write +that tears the directory's *links* is not, and the membership check in +``Stripe::_shm_directory_is_valid`` is what catches it. The on-disk directory is still written -------------------------------------- diff --git a/src/iocore/cache/CacheDir.cc b/src/iocore/cache/CacheDir.cc index a5746297a20..5cf4b92b443 100644 --- a/src/iocore/cache/CacheDir.cc +++ b/src/iocore/cache/CacheDir.cc @@ -27,7 +27,6 @@ #include "P_CacheInternal.h" #include "PreservationTable.h" #include "Stripe.h" -#include "CacheShm.h" #include "tscore/hugepages.h" #include "tscore/Random.h" @@ -958,11 +957,8 @@ sync_cache_dir_on_shutdown() thr.join(); } - // Each stripe was snapshotted above. This does not stop future writers (the event system - // is still up), but a late or in-flight write only leaves a dir entry the next start's - // magic+key read check treats as a miss -- never served corruption -- so mark clean here. - CacheShm::mark_clean_shutdown(); - + // The event system is still up here, so the shm trust flag is not set yet -- the caller marks it clean only + // after shutting the event system down. See CacheShm::mark_clean_shutdown. Dbg(dbg_ctl_cache_dir_sync, "shutdown sync done"); } diff --git a/src/iocore/cache/CacheShm.cc b/src/iocore/cache/CacheShm.cc index 1fab3327213..157a5097b54 100644 --- a/src/iocore/cache/CacheShm.cc +++ b/src/iocore/cache/CacheShm.cc @@ -885,10 +885,10 @@ CacheShm::mark_clean_shutdown() } Note("cache shm: marking clean shutdown"); g_control->clean_shutdown = 1; - // owner_pid stays until the process exits. Event threads are still running here (see sync_cache_dir_on_shutdown), and - // where flock is a no-op -- macOS, FreeBSD -- it is the only thing that stops a starting process from attaching to - // directories we may still be writing. A stale PID only costs the next start its fast restart; a concurrent attach - // costs correctness. + // owner_pid stays until the process exits. The event system is shut down by the time we get here, but that flag joins + // no thread, and where flock is a no-op -- macOS, FreeBSD -- the PID is the only thing that stops a starting process + // from attaching to directories a straggler may still be writing. A stale PID only costs the next start its fast + // restart; a concurrent attach costs correctness. msync(g_control, CONTROL_SIZE, MS_SYNC); } diff --git a/src/iocore/cache/CacheShm.h b/src/iocore/cache/CacheShm.h index 751f508bfd6..63f87a1c2a5 100644 --- a/src/iocore/cache/CacheShm.h +++ b/src/iocore/cache/CacheShm.h @@ -62,7 +62,8 @@ class CacheShm /// Whether a pointer was returned from attach_or_create_stripe (munmap vs ats_free). static bool is_shm_pointer(char *raw_dir); - /// Called after sync_cache_dir_on_shutdown; a crash instead leaves the flag clear, which drops the segment next start. + /// Called once the event system is shut down, after sync_cache_dir_on_shutdown; a crash instead leaves the flag clear, + /// which drops the segment next start. static void mark_clean_shutdown(); /// Marks this stripe's control-segment entry so the next start recreates it rather than attaching. Never writes through diff --git a/src/iocore/cache/Stripe.cc b/src/iocore/cache/Stripe.cc index cf647c28044..bbebee525d2 100644 --- a/src/iocore/cache/Stripe.cc +++ b/src/iocore/cache/Stripe.cc @@ -208,6 +208,8 @@ Stripe::_shm_directory_is_valid() const int64_t segment_entries = static_cast(this->directory.buckets) * DIR_DEPTH; + std::vector visited; + for (int s = 0; s < this->directory.segments; s++) { if (this->directory.header->freelist[s] >= segment_entries) { return false; @@ -233,7 +235,7 @@ Stripe::_shm_directory_is_valid() } } - if (!this->_shm_freelist_is_valid(s, seg, segment_entries)) { + if (!this->_shm_segment_membership_is_valid(s, seg, segment_entries, visited)) { return false; } @@ -245,17 +247,25 @@ Stripe::_shm_directory_is_valid() return true; } -// The per-entry bounds above only prove each link points inside the segment; an in-range cycle or a stale prev link -// still passes, and Directory::check_segment() walks the bucket chains, not the free list. +// The per-entry bounds above only prove each link points inside the segment; they say nothing about the shape. Prove +// that too: every entry must be reached exactly once, as a bucket root, as an empty free-list node, or as an in-use +// bucket-chain node. Reachability alone is not enough -- a shutdown torn mid-Directory::insert leaves an entry unlinked +// from the free list but not yet filled, so no walk visits it, and the next insert to find that empty row writes +// through its stale prev/next into a live chain or over a live entry's tag. bool -Stripe::_shm_freelist_is_valid(int s, Dir *seg, int64_t segment_entries) +Stripe::_shm_segment_membership_is_valid(int s, Dir *seg, int64_t segment_entries, std::vector &visited) { - int64_t node = this->directory.header->freelist[s]; - int64_t prev = 0; - int64_t steps = 0; + // Below 5, a link is a raw segment index; at or above it dir_from_offset() compacts roots out of the link space and + // the raw dir_in_seg() indexing here (and in the bounds loop above) would address the wrong entries. + static_assert(DIR_DEPTH < 5, "Dir link values are treated as raw segment indices"); + + visited.assign(segment_entries, false); + + int64_t node = this->directory.header->freelist[s]; + int64_t prev = 0; while (node != 0) { - if (node >= segment_entries || ++steps > segment_entries) { + if (node >= segment_entries || visited[node]) { return false; } Dir *e = dir_in_seg(seg, node); @@ -265,8 +275,39 @@ Stripe::_shm_freelist_is_valid(int s, Dir *seg, int64_t segment_entries) if (!dir_is_empty(e) || dir_prev(e) != prev) { return false; } - prev = node; - node = dir_next(e); + visited[node] = true; + prev = node; + node = dir_next(e); + } + + for (int64_t b = 0; b < this->directory.buckets; b++) { + const int64_t root = b * DIR_DEPTH; + + // A root is reachable only as a root: init_segment() frees rows 1..DIR_DEPTH-1 onto the free list, never row 0. + if (visited[root]) { + return false; + } + visited[root] = true; + + // A chain carries no back-links to check, since dir_prev is tag/phase/head/pinned on the in-use entries it holds. + node = dir_next(dir_in_seg(seg, root)); + while (node != 0) { + if (node >= segment_entries || node % DIR_DEPTH == 0 || visited[node]) { + return false; + } + Dir *e = dir_in_seg(seg, node); + if (dir_is_empty(e)) { + return false; + } + visited[node] = true; + node = dir_next(e); + } + } + + for (int64_t i = 0; i < segment_entries; i++) { + if (!visited[i]) { + return false; + } } return true; diff --git a/src/iocore/cache/Stripe.h b/src/iocore/cache/Stripe.h index e863e8ace47..77b29c2fec1 100644 --- a/src/iocore/cache/Stripe.h +++ b/src/iocore/cache/Stripe.h @@ -35,6 +35,7 @@ #include #include +#include #define CACHE_BLOCK_SHIFT 9 #define CACHE_BLOCK_SIZE (1 << CACHE_BLOCK_SHIFT) // 512, smallest sector size @@ -158,7 +159,7 @@ class Stripe void _init_data(off_t store_block_size, int avg_obj_size = -1); void _init_data_internal(int avg_obj_size = -1); // Defaults to cache_config_min_average_object_size; void _init_directory(std::size_t directory_size, int header_size, int footer_size); - bool _shm_freelist_is_valid(int s, Dir *seg, int64_t segment_entries); + bool _shm_segment_membership_is_valid(int s, Dir *seg, int64_t segment_entries, std::vector &visited); }; inline uint32_t diff --git a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc index ff5c53787dc..cfc1024959a 100644 --- a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc +++ b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc @@ -393,6 +393,95 @@ TEST_CASE("A cyclic segment free list fails the attach gate", "[cache][shm]") unlink_test_segments(); } +// What a walk from the free-list head cannot see. Directory::insert() unlinks an empty row from the free list and only +// then fills it; a shutdown torn in between leaves that row empty, off the free list, out of every bucket chain, and still +// holding its old links -- so the next insert to pick it writes through them into a live chain. +TEST_CASE("An entry unlinked from the free list but never filled fails the attach gate", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + disk.hw_sector_size = 512; + attach_tmpfile_to_stripe(stripe); + stripe.clear_dir(); + REQUIRE(stripe.shm_directory_is_valid()); + + Dir *seg = stripe.directory.get_segment(0); + const uint16_t head = stripe.directory.header->freelist[0]; + Dir *torn = dir_in_seg(seg, dir_next(dir_in_seg(seg, head))); + REQUIRE(dir_next(dir_in_seg(seg, head)) != 0); + REQUIRE(dir_next(torn) != 0); + + // Exactly what Directory::unlink_from_freelist() does, leaving torn's own links untouched and its offset still 0. + dir_set_next(dir_in_seg(seg, dir_prev(torn)), dir_next(torn)); + dir_set_prev(dir_in_seg(seg, dir_next(torn)), dir_prev(torn)); + + // The free list that remains is self-consistent, so a reachability-only walk of it sees nothing wrong. + REQUIRE(dir_is_empty(torn)); + REQUIRE(dir_next(torn) < static_cast(stripe.directory.buckets) * DIR_DEPTH); + REQUIRE(dir_prev(torn) < static_cast(stripe.directory.buckets) * DIR_DEPTH); + + CHECK_FALSE(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + +// The other half of the same tear: torn after the row is linked into the bucket chain but before dir_assign_data fills it. +// The row belongs to exactly one structure, so membership alone accepts it; probe() would then walk an offset-0 entry. +TEST_CASE("An empty entry linked into a bucket chain fails the attach gate", "[cache][shm]") +{ + unlink_test_segments(); + if (!enable_shm()) { + WARN("shm unavailable in this environment; skipping"); + unlink_test_segments(); + return; + } + + CacheDisk disk; + init_disk(disk); + CacheVol cache_vol; + { + GateStripe stripe{&disk, 10, 0}; + stripe.cache_vol = &cache_vol; + disk.hw_sector_size = 512; + attach_tmpfile_to_stripe(stripe); + stripe.clear_dir(); + REQUIRE(stripe.shm_directory_is_valid()); + + Dir *seg = stripe.directory.get_segment(0); + Dir *root = dir_in_seg(seg, 0); + dir_set_offset(root, 1); + dir_set_head(root, 1); + REQUIRE(stripe.shm_directory_is_valid()); + + // Pop the head cleanly, the way freelist_pop() does, then link it in without filling it. + const uint16_t head = stripe.directory.header->freelist[0]; + Dir *e = dir_in_seg(seg, head); + stripe.directory.header->freelist[0] = dir_next(e); + dir_set_prev(dir_in_seg(seg, dir_next(e)), 0); + dir_set_next(e, 0); + dir_set_next(root, head); + + REQUIRE(dir_is_empty(e)); + REQUIRE(stripe.directory.check_segment(0)); + + CHECK_FALSE(stripe.shm_directory_is_valid()); + } + + unlink_test_segments(); +} + // What the flush-failure path actually leans on. A real short write leaves write_pos in range with agg_pos ahead of it, so // the quiesced-cursor check has to be what rejects the segment -- not the range check, which only a synthetic write_pos // trips. diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index f253c0cdead..1f174fb1810 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -86,6 +86,7 @@ extern "C" int plock(int); #include "../iocore/dns/P_SplitDNSProcessor.h" #include "../iocore/hostdb/P_HostDB.h" #include "../iocore/cache/P_CacheDir.h" +#include "../iocore/cache/CacheShm.h" #include "../records/P_RecCore.h" #include "tscore/Layout.h" #include "iocore/utils/Machine.h" @@ -301,6 +302,13 @@ struct AutoStopCont : public Continuation { TSSystemState::shut_down_event_system(); + // Only now is the directory quiescent enough to be worth trusting next start; marking clean while event threads + // could still mutate it would publish a directory torn mid-Directory::insert. Not a hard barrier -- the flag above + // stops new work but joins no thread -- so Stripe::_shm_directory_is_valid still has to prove the structure. + if (cacheProcessor.IsCacheEnabled() == CacheInitState::INITIALIZED) { + CacheShm::mark_clean_shutdown(); + } + // Wake preproc threads to drain remaining log buffers before exit. for (int i = 0; i < Log::preproc_threads; i++) { Log::preproc_notify[i].signal(); From cd4bff914db0e74ad4093b10d830808f816499b1 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 7 Aug 2026 09:59:46 +0900 Subject: [PATCH 7/7] Include algorithm header --- src/traffic_ctl/CacheShmCommand.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/traffic_ctl/CacheShmCommand.cc b/src/traffic_ctl/CacheShmCommand.cc index db5fc612f17..d412f8c484d 100644 --- a/src/traffic_ctl/CacheShmCommand.cc +++ b/src/traffic_ctl/CacheShmCommand.cc @@ -39,6 +39,7 @@ #include #include +#include #include #include #include